diff --git a/pom.xml b/pom.xml
index 5232d15..0198b42 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1,141 +1,169 @@
- 4.0.0
+ xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd">
+ 4.0.0
-
- maven-parent
- org.gcube.tools
- 1.0.0
-
-
+
+ maven-parent
+ org.gcube.tools
+ 1.0.0
+
+
- org.gcube.dataAnalysis
- dataminer-pool-manager
- 2.0.0-SNAPSHOT
- war
+ org.gcube.dataAnalysis
+ dataminer-pool-manager
+ 2.0.0-SNAPSHOT
+ war
- dataminer-pool-manager
-
-
+ dataminer-pool-manager
+
+
-
- distro
- UTF-8
-
+
+ distro
+ UTF-8
+ 1.5.13
+ 2.25.1
+
-
-
- org.gcube.core
- common-smartgears
-
- provided
-
-
- commons-io
- commons-io
- [2.5.0,2.6.0)
-
-
- org.antlr
- stringtemplate
- [4.0.0, 4.1.0)
-
-
- org.tmatesoft.svnkit
- svnkit
- 1.8.5
-
-
- commons-configuration
- commons-configuration
- 1.10
-
+
-
- net.sf.opencsv
- opencsv
- 2.3
-
-
- org.gcube.resources.discovery
- ic-client
- provided
-
-
- org.yaml
- snakeyaml
- 1.16
-
-
- org.glassfish.jersey.media
- jersey-media-json-jackson
- 2.23.1
-
-
- org.glassfish.jersey.core
- jersey-client
- 2.22.1
-
-
- org.glassfish.jersey.containers
- jersey-container-servlet
- 2.22.1
-
+
+ org.gcube.core
+ common-smartgears
+ provided
+
-
- org.slf4j
- slf4j-api
- provided
-
+
+ org.gcube.resources.discovery
+ ic-client
+ provided
+
-
-
-
-
- org.gcube.distribution
- maven-smartgears-bom
- LATEST
- pom
- import
-
-
-
+
+ commons-io
+ commons-io
+ [2.5.0,2.6.0)
+
+
+ org.antlr
+ stringtemplate
+ [4.0.0, 4.1.0)
+
+
+ org.tmatesoft.svnkit
+ svnkit
+ 1.8.5
+
+
+ commons-configuration
+ commons-configuration
+ 1.10
+
-
-
+
+ net.sf.opencsv
+ opencsv
+ 2.3
+
-
- maven-compiler-plugin
-
-
- 1.7
-
-
+
+ org.yaml
+ snakeyaml
+ 1.16
+
+
+ org.glassfish.jersey.media
+ jersey-media-json-jackson
+ $***REMOVED***version.jersey***REMOVED***
+
-
- org.apache.maven.plugins
- maven-assembly-plugin
-
-
- $***REMOVED***distroDirectory***REMOVED***/descriptor.xml
-
-
-
-
- servicearchive
- install
-
- single
-
-
-
-
+
+ org.glassfish.jersey.core
+ jersey-client
+ $***REMOVED***version.jersey***REMOVED***
+
-
-
+
+ org.glassfish.jersey.containers
+ jersey-container-servlet
+ $***REMOVED***version.jersey***REMOVED***
+
+
+
+
+ org.slf4j
+ slf4j-api
+ provided
+
+
+
+
+
+
+
+
+ org.gcube.distribution
+ maven-smartgears-bom
+ LATEST
+ pom
+ import
+
+
+
+
+
+
+
+
+ org.apache.maven.plugins
+ maven-assembly-plugin
+
+
+ $***REMOVED***distroDirectory***REMOVED***/descriptor.xml
+
+
+
+
+ servicearchive
+ install
+
+ single
+
+
+
+
+
+
+
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java
index 60b9cb8..b4b616a 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java
@@ -1,6 +1,8 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection;
+import java.util.HashSet;
+***REMOVED***
import java.util.Vector;
@@ -76,6 +78,36 @@ public class Algorithm ***REMOVED***
public Collection getDependencies() ***REMOVED***
return dependencies;
***REMOVED***
+
+ public Collection getOSDependencies()***REMOVED***
+ Set deps = new HashSet<>();
+ for(Dependency d: this.getDependencies())***REMOVED***
+ if(d.getType().equals("os"))***REMOVED***
+ deps.add(d);
+ ***REMOVED***
+ ***REMOVED***
+ return deps;
+ ***REMOVED***
+
+ public Collection getCranDependencies()***REMOVED***
+ Set deps = new HashSet<>();
+ for(Dependency d: this.getDependencies())***REMOVED***
+ if(d.getType().equals("cran"))***REMOVED***
+ deps.add(d);
+ ***REMOVED***
+ ***REMOVED***
+ return deps;
+ ***REMOVED***
+
+ public Collection getGitHubDependencies()***REMOVED***
+ Set deps = new HashSet<>();
+ for(Dependency d: this.getDependencies())***REMOVED***
+ if(d.getType().equals("github"))***REMOVED***
+ deps.add(d);
+ ***REMOVED***
+ ***REMOVED***
+ return deps;
+ ***REMOVED***
public void setDependencies(Collection deps) ***REMOVED***
this.dependencies = deps;
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java
index aa69411..4d7f9a4 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java
@@ -74,27 +74,27 @@ public abstract class DMPMJob ***REMOVED***
PrintStream ps = new PrintStream(fos);
return worker.execute(ps);
***REMOVED***
-
-
- protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED***
- for (Dependency d : algo.getDependencies()) ***REMOVED***
-
- if (d.getType().equals("os")) ***REMOVED***
- List ls = new LinkedList();
- ls.add(d.getName());
- this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
- ***REMOVED***
- if (d.getType().equals("cran")) ***REMOVED***
- List ls = new LinkedList();
- ls.add(d.getName());
- this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
- ***REMOVED***
- if (d.getType().equals("github")) ***REMOVED***
- List ls = new LinkedList();
- ls.add(d.getName());
- this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
- ***REMOVED***
- ***REMOVED***
-***REMOVED***
+
+***REMOVED***
+***REMOVED*** protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED***
+***REMOVED*** for (Dependency d : algo.getDependencies()) ***REMOVED***
+***REMOVED***
+***REMOVED*** if (d.getType().equals("os")) ***REMOVED***
+***REMOVED*** List ls = new LinkedList();
+***REMOVED*** ls.add(d.getName());
+***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
+***REMOVED*** ***REMOVED***
+***REMOVED*** if (d.getType().equals("cran")) ***REMOVED***
+***REMOVED*** List ls = new LinkedList();
+***REMOVED*** ls.add(d.getName());
+***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
+***REMOVED*** ***REMOVED***
+***REMOVED*** if (d.getType().equals("github")) ***REMOVED***
+***REMOVED*** List ls = new LinkedList();
+***REMOVED*** ls.add(d.getName());
+***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
+***REMOVED*** ***REMOVED***
+***REMOVED*** ***REMOVED***
+***REMOVED******REMOVED***
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
index 98f2d6c..f3ad2ec 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
@@ -5,10 +5,11 @@ import java.io.FileNotFoundException;
***REMOVED***
import java.util.Scanner;
+***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
-import org.gcube.dataanalysis.dataminer.poolmanager.util.Props;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
***REMOVED***
@@ -20,7 +21,7 @@ public class DataminerPoolManager ***REMOVED***
public DataminerPoolManager() ***REMOVED***
try ***REMOVED***
***REMOVED***TODO: read this from configuration
- this.svnUpdater = new SVNUpdater(new Props().getSVNrepo());
+ this.svnUpdater = new SVNUpdater(new ServiceConfiguration());
***REMOVED*** catch (SVNException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
@@ -33,7 +34,7 @@ public class DataminerPoolManager ***REMOVED***
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
- DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster);
+ DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get());
String id = job.start();
return id;
***REMOVED***
@@ -42,7 +43,7 @@ public class DataminerPoolManager ***REMOVED***
Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
- DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster);
+ DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE);
String id = job.start();
return id;
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
index ca3c1d5..359aefa 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
@@ -8,20 +8,24 @@ public class ProductionPublishingJob extends DMPMJob ***REMOVED***
private Algorithm algorithm;
private Cluster prodCluster;
+ private String targetVREName;
- public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster) ***REMOVED***
+ public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName) ***REMOVED***
super(svnUpdater);
this.algorithm = algorithm;
this.prodCluster = prodCluster;
+ this.targetVREName = targetVREName;
***REMOVED***
***REMOVED***
protected void execute() ***REMOVED***
try ***REMOVED***
- this.updateSVNDependencies(this.algorithm, false);
+ this.svnUpdater.updateProdDeps(this.algorithm);
- int ret = this.executeAnsibleWorker(
- createWorker(this.algorithm, this.prodCluster, false, "gcube"));
+ this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
+
+***REMOVED*** int ret = this.executeAnsibleWorker(
+***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube"));
***REMOVED*** catch (Exception e) ***REMOVED***
e.printStackTrace();
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
index b958a0e..1567dad 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
@@ -10,12 +10,19 @@ public class StagingJob extends DMPMJob ***REMOVED***
private Algorithm algorithm;
private Cluster stagingCluster;
private Cluster rProtoCluster;
+ private String rProtoVREName;
- public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster) ***REMOVED***
+ public StagingJob(
+ SVNUpdater svnUpdater,
+ Algorithm algorithm,
+ Cluster stagingCluster,
+ Cluster rProtoCluster,
+ String rProtoVREName) ***REMOVED***
super(svnUpdater);
this.algorithm = algorithm;
this.stagingCluster = stagingCluster;
this.rProtoCluster = rProtoCluster;
+ this.rProtoVREName = rProtoVREName;
***REMOVED***
***REMOVED***
@@ -26,13 +33,13 @@ public class StagingJob extends DMPMJob ***REMOVED***
createWorker(this.algorithm, this.stagingCluster, true, "root"));
if(ret == 0)***REMOVED***
+
+ this.svnUpdater.updateRPRotoDeps(this.algorithm);
+
+ this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, "DataMiner Pool Manager", "Proto");
-
- this.updateSVNDependencies(this.algorithm, true);
-
-
- int ret2 = this.executeAnsibleWorker(
- createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
+***REMOVED*** int ret2 = this.executeAnsibleWorker(
+***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
index c96df6b..46d21f9 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
@@ -15,7 +15,7 @@ public class ClusterBuilder ***REMOVED***
***REMOVED***1. to complete
public static Cluster getStagingDataminerCluster() throws FileNotFoundException***REMOVED***
Cluster cluster = new Cluster();
- Props p = new Props();
+ ServiceConfiguration p = new ServiceConfiguration();
Host h = new Host(p.getStagingHost());
***REMOVED***TODO: read this from configuration or IS?
h.setName(p.getStagingHost());
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/Props.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/Props.java
deleted file mode 100644
index 033f7e7..0000000
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/Props.java
+++ /dev/null
@@ -1,103 +0,0 @@
-package org.gcube.dataanalysis.dataminer.poolmanager.util;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-***REMOVED***
-import java.io.InputStream;
-import java.util.Properties;
-
-public class Props ***REMOVED***
- public static String home = System.getProperty("user.home");
-
- public Props() ***REMOVED***
-***REMOVED***
-
- public String getCSVUrl() ***REMOVED***
- Properties props = new Properties();
- ***REMOVED***read from war
- ***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
-
- ***REMOVED***read from filesystem
- FileInputStream input;
- try ***REMOVED***
- input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
- ***REMOVED*** loading properites from properties file
- try ***REMOVED***
- props.load(input);
- ***REMOVED*** catch (IOException e) ***REMOVED***
- ***REMOVED*** TODO Auto-generated catch block
- e.printStackTrace();
- ***REMOVED***
- ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
- ***REMOVED*** TODO Auto-generated catch block
- e1.printStackTrace();
- ***REMOVED***
- ***REMOVED*** reading propeprty
- String path = props.getProperty("HAPROXY_CSV");
- return path;
-
-***REMOVED***
-
-
- public String getSVNrepo() ***REMOVED***
- Properties props = new Properties();
- ***REMOVED***read from war
- ***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
-
- ***REMOVED***read from filesystem
- FileInputStream input;
- try ***REMOVED***
- input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
- ***REMOVED*** loading properites from properties file
- try ***REMOVED***
- props.load(input);
- ***REMOVED*** catch (IOException e) ***REMOVED***
- ***REMOVED*** TODO Auto-generated catch block
- e.printStackTrace();
- ***REMOVED***
- ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
- ***REMOVED*** TODO Auto-generated catch block
- e1.printStackTrace();
- ***REMOVED***
- ***REMOVED*** reading propeprty
- String path = props.getProperty("SVN_REPO");
- return path;
-
-***REMOVED***
-
-
-
- public String getStagingHost() ***REMOVED***
- Properties props = new Properties();
- ***REMOVED***read from war
- ***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
-
- ***REMOVED***read from filesystem
- FileInputStream input;
- try ***REMOVED***
- input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
- ***REMOVED*** loading properites from properties file
- try ***REMOVED***
- props.load(input);
- ***REMOVED*** catch (IOException e) ***REMOVED***
- ***REMOVED*** TODO Auto-generated catch block
- e.printStackTrace();
- ***REMOVED***
- ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
- ***REMOVED*** TODO Auto-generated catch block
- e1.printStackTrace();
- ***REMOVED***
-
- ***REMOVED*** reading propeprty
- String path = props.getProperty("STAGING_HOST");
- return path;
-
-***REMOVED***
-
- public static void main(String[] args) throws FileNotFoundException ***REMOVED***
- Props a = new Props();
- System.out.println(new Props().getSVNrepo());
- System.out.println(a.getStagingHost());
- System.out.println(a.getCSVUrl());
-***REMOVED***
-***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
index c40d615..232588c 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
@@ -1,5 +1,7 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
***REMOVED***
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
@@ -21,32 +23,117 @@ import java.util.*;
*/
public class SVNUpdater ***REMOVED***
-
private SVNRepository svnRepository;
+ private ServiceConfiguration configuration;
- public SVNUpdater(String rootURL) throws SVNException ***REMOVED***
- this.svnRepository = this.getSvnRepository(
- rootURL);
- ***REMOVED***
-
-
- private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED***
- SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
+ public SVNUpdater(ServiceConfiguration configuration) throws SVNException ***REMOVED***
+ this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
- repository.setAuthenticationManager(authManager);
- System.out.println(repository.getLocation());
-
- return repository;
+ this.svnRepository.setAuthenticationManager(authManager);
+ this.configuration = configuration;
***REMOVED***
+ public void updateRPRotoDeps(Algorithm algorithm)***REMOVED***
+ this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies());
+ this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies());
+ this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
+ ***REMOVED***
- public void updateSVN(String file, List ldep) ***REMOVED***
+ public void updateProdDeps(Algorithm algorithm)***REMOVED***
+ this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
+ this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
+ this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
+ ***REMOVED***
+
+
+ public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
+ this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env);
+ ***REMOVED***
+
+
+ public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
+ this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env);
+ ***REMOVED***
+
+ public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
try ***REMOVED***
+ System.out.println("Updating algorithm list: "+ file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
+
+ ***REMOVED***check if the algorithm is already in the list (match the class name)
+ for(String l: lines)***REMOVED***
+ if(l.contains(algorithm.getClazz()))***REMOVED***
+ return;
+ ***REMOVED***
+ ***REMOVED***
+
+ ***REMOVED***the algorithm is not in the list. Add it
+ List newContent = new LinkedList<>(Arrays.asList(lines));
+ newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, user, env));
+ Collections.sort(newContent);
+
+
+ final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
+
+ byte[] originalContents = byteArrayOutputStream.toByteArray();
+
+ final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
+ commitEditor.openRoot(-1);
+ commitEditor.openFile(file, -1);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ for (String line : newContent) ***REMOVED***
+ baos.write(line.getBytes());
+ baos.write("\n".getBytes());
+ ***REMOVED***
+ byte[] bytes = baos.toByteArray();
+
+ commitEditor.applyTextDelta(file,md5(originalContents));
+
+ final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
+ new ByteArrayInputStream(bytes), commitEditor, true);
+ commitEditor.closeFile(file, checksum);
+ commitEditor.closeEdit();
+
+
+ ***REMOVED***catch(Exception ex)***REMOVED***
+ ex.printStackTrace();
+ ***REMOVED***
+
+ finally ***REMOVED***
+ svnRepository.closeSession();
+ ***REMOVED*** ***REMOVED***
+
+
+ public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
+ StringBuffer sb = new StringBuffer("| ");
+ sb.append(algorithm.getName() +" | ");
+ sb.append(user + " | ");
+ sb.append(algorithm.getCategory() + " | ");
+ sb.append(env + " | ");
+ sb.append("./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | ");
+ sb.append("none |");
+ return sb.toString();
+ ***REMOVED***
+
+
+ public void updateSVN(String file, Collection deps) ***REMOVED***
+ try ***REMOVED***
+ System.out.println("Updating dependencies list: "+ file);
+
+ final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
+ String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
+
+ List ldep = new LinkedList<>();
+ for(Dependency d: deps)***REMOVED***
+ ldep.add(d.getName());
+ ***REMOVED***
List aa = this.checkMatch(lines, ldep);
Collections.sort(aa);
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java
new file mode 100644
index 0000000..954baac
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java
@@ -0,0 +1,91 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+***REMOVED***
+import java.util.Properties;
+
+public class ServiceConfiguration ***REMOVED***
+ public static String home = System.getProperty("user.home");
+
+ private Properties props;
+
+ public ServiceConfiguration() ***REMOVED***
+ this(home+"/dataminer-pool-manager/dpmConfig/service.properties");
+***REMOVED***
+
+ public ServiceConfiguration(String configFile) ***REMOVED***
+ this.props = new Properties();
+
+ FileInputStream input;
+
+ try ***REMOVED***
+ input = new FileInputStream(configFile);
+ ***REMOVED*** loading properites from properties file
+ try ***REMOVED***
+ props.load(input);
+ ***REMOVED*** catch (IOException e) ***REMOVED***
+ ***REMOVED*** TODO Auto-generated catch block
+ e.printStackTrace();
+ ***REMOVED***
+ ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
+ ***REMOVED*** TODO Auto-generated catch block
+ e1.printStackTrace();
+ ***REMOVED***
+***REMOVED***
+
+
+
+ public String getSVNRepository()***REMOVED***
+ return props.getProperty("svn.repository");
+***REMOVED***
+
+ public String getSVNRProtoAlgorithmsList()***REMOVED***
+ return props.getProperty("svn.rproto.algorithms-list");
+***REMOVED***
+
+ public String getSVNRProtoOSDepsList()***REMOVED***
+ return props.getProperty("svn.rproto.deps-os");
+***REMOVED***
+
+ public String getSVNRProtoCRANDepsList()***REMOVED***
+ return props.getProperty("svn.rproto.deps-cran");
+***REMOVED***
+
+ public String getSVNRProtoGitHubDepsList()***REMOVED***
+ return props.getProperty("svn.rproto.deps-github");
+***REMOVED***
+
+ public String getSVNProdAlgorithmsList()***REMOVED***
+ return props.getProperty("svn.prod.algorithms-list");
+***REMOVED***
+
+ public String getSVNProdOSDepsList()***REMOVED***
+ return props.getProperty("svn.prod.deps-os");
+***REMOVED***
+
+ public String getSVNRProdCRANDepsList()***REMOVED***
+ return props.getProperty("svn.prod.deps-cran");
+***REMOVED***
+
+ public String getSVNRProdGitHubDepsList()***REMOVED***
+ return props.getProperty("svn.prod.deps-github");
+***REMOVED***
+
+ public String getCSVUrl() ***REMOVED***
+ return props.getProperty("HAPROXY_CSV");
+***REMOVED***
+
+
+ public String getStagingHost() ***REMOVED***
+ return props.getProperty("STAGING_HOST");
+***REMOVED***
+
+ public static void main(String[] args) throws FileNotFoundException ***REMOVED***
+ ServiceConfiguration a = new ServiceConfiguration();
+ System.out.println(a.getStagingHost());
+ System.out.println(a.getCSVUrl());
+***REMOVED***
+
+
+***REMOVED***
diff --git a/src/main/resources/service.properties b/src/main/resources/service.properties
index 915900b..498c796 100644
--- a/src/main/resources/service.properties
+++ b/src/main/resources/service.properties
@@ -3,3 +3,17 @@
STAGING_HOST: dataminer1-devnext.d4science.org
SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
#HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0
+
+svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube
+
+svn.rproto.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/proto/algorithms
+svn.rproto.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_deb_pkgs.txt
+svn.rproto.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_cran_pkgs.txt
+svn.rproto.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_github_pkgs.txt
+
+
+svn.prod.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/prod/algorithms
+svn.prod.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt
+svn.prod.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
+svn.prod.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt
+
diff --git a/src/main/webapp/WEB-INF/gcube-app.xml b/src/main/webapp/WEB-INF/gcube-app.xml
index 15f231c..204fb09 100644
--- a/src/main/webapp/WEB-INF/gcube-app.xml
+++ b/src/main/webapp/WEB-INF/gcube-app.xml
@@ -9,5 +9,7 @@
-
+
+ /api/swagger.*
+
\ No newline at end of file
diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml
index 2a231cb..5596e2b 100644
--- a/src/main/webapp/WEB-INF/web.xml
+++ b/src/main/webapp/WEB-INF/web.xml
@@ -11,14 +11,32 @@
org.glassfish.jersey.servlet.ServletContainer
jersey.config.server.provider.packages
- org.gcube.dataanalysis.dataminer.poolmanager.rest
+
+ io.swagger.jaxrs.listing,
+ org.gcube.dataanalysis.dataminer.poolmanager.rest
1
REST-API
- /rest/*
+ /api/*
-
+
+
+
diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java
index 7d2e20b..096d8d1 100644
--- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java
+++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java
@@ -11,6 +11,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.service.StagingJob;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
***REMOVED***
public class JobTest ***REMOVED***
@@ -19,13 +20,13 @@ public class JobTest ***REMOVED***
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
- SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
+ SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration());
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
***REMOVED***test phase
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
- DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster);
+ DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get());
job.start();
***REMOVED***release phase
diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java
new file mode 100644
index 0000000..54a82f5
--- /dev/null
+++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java
@@ -0,0 +1,26 @@
+package org.gcube.dataanalysis.dataminerpoolmanager;
+
+***REMOVED***
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
+***REMOVED***
+
+***REMOVED***
+
+/**
+ * Created by ggiammat on 5/17/17.
+ */
+public class SVNTests ***REMOVED***
+
+
+ public static void main(String[] args) throws SVNException, IOException, InterruptedException ***REMOVED***
+
+
+ SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ggiammat/tmp/dmpm.properties"));
+ Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
+
+
+ svnUpdater.updateRPRotoDeps(algo);
+ ***REMOVED***
+***REMOVED***
diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java
deleted file mode 100644
index d9c4d04..0000000
--- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package org.gcube.dataanalysis.dataminerpoolmanager;
-
-***REMOVED***
-import java.util.LinkedList;
-***REMOVED***
-
-***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
-***REMOVED***
-
-public class SVNUpdaterTest ***REMOVED***
-
- public static void main(String[] args) throws SVNException, IOException ***REMOVED***
- ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
-
- SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
-
- String test = "testDep";
- List ldep = new LinkedList<>();
- ldep.add(test);
-
- svnUpdater.updateSVN("test_r_cran_pkgs.txt", ldep);
-***REMOVED***
-
-***REMOVED***