gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@154344 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-09-20 16:51:21 +00:00
parent 17c944b4de
commit a951066ac8
10 changed files with 584 additions and 413 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -42,7 +42,7 @@ public class RestPoolManager implements PoolManager {
private DataminerPoolManager service = new DataminerPoolManager(); private DataminerPoolManager service = new DataminerPoolManager();
***REMOVED***@Context ***REMOVED***@Context
private ApplicationContext context = ContextProvider.get(); ***REMOVED***private ApplicationContext context = ContextProvider.get();
@ -55,8 +55,8 @@ public class RestPoolManager implements PoolManager {
@QueryParam("category") String category, @QueryParam("category") String category,
@QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException { @QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException {
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
String env = context.application().getInitParameter("Environment"); ***REMOVED***String env = context.application().getInitParameter("Environment");
return this.service.stageAlgorithm(algo,targetVRE,category,algorithm_type,env); return this.service.stageAlgorithm(algo,targetVRE,category,algorithm_type/*,env*/);
***REMOVED*** ***REMOVED***
@ -70,8 +70,8 @@ public class RestPoolManager implements PoolManager {
@QueryParam("category") String category, @QueryParam("category") String category,
@QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException { @QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException {
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
String env = context.application().getInitParameter("Environment"); ***REMOVED***String env = context.application().getInitParameter("Environment");
return this.service.publishAlgorithm(algo, /*targetVREToken,*/ targetVRE,category,algorithm_type,env); return this.service.publishAlgorithm(algo, /*targetVREToken,*/ targetVRE,category,algorithm_type/*,env*/);
***REMOVED*** ***REMOVED***
/* /*

10
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -28,23 +28,23 @@ public class DataminerPoolManager {
***REMOVED*** ***REMOVED***
public String stageAlgorithm(Algorithm algo,String targetVRE, String category, String algorithm_type,String env) throws IOException, InterruptedException { public String stageAlgorithm(Algorithm algo,String targetVRE, String category, String algorithm_type/*,String env*/) throws IOException, InterruptedException {
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(env); Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); ***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type,env); DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/);
String id = job.start(); String id = job.start();
return id; return id;
***REMOVED*** ***REMOVED***
public String publishAlgorithm(Algorithm algo, String targetVRE, String category, String algorithm_type, String env) throws IOException, InterruptedException { public String publishAlgorithm(Algorithm algo, String targetVRE, String category, String algorithm_type/*, String env*/) throws IOException, InterruptedException {
***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); ***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, /*prodCluster,*/ targetVRE, category, algorithm_type,env); DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, /*prodCluster,*/ targetVRE, category, algorithm_type/*,env*/);
String id = job.start(); String id = job.start();
return id; return id;
***REMOVED*** ***REMOVED***

8
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
View File

@ -22,11 +22,11 @@ public class ProductionPublishingJob extends DMPMJob {
private String category; private String category;
private String algorithm_type; private String algorithm_type;
***REMOVED***private String targetVREToken; ***REMOVED***private String targetVREToken;
private String env; ***REMOVED***private String env;
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm,
/*Cluster prodCluster,*/ String targetVREName, String category,String algorithm_type, String env) throws FileNotFoundException, UnsupportedEncodingException { /*Cluster prodCluster,*/ String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException {
super(svnUpdater); super(svnUpdater);
this.algorithm = algorithm; this.algorithm = algorithm;
***REMOVED***this.prodCluster = prodCluster; ***REMOVED***this.prodCluster = prodCluster;
@ -34,7 +34,7 @@ public class ProductionPublishingJob extends DMPMJob {
this.category = category; this.category = category;
this.algorithm_type = algorithm_type; this.algorithm_type = algorithm_type;
***REMOVED***this.targetVREToken = targetVREToken; ***REMOVED***this.targetVREToken = targetVREToken;
this.env= env; ***REMOVED***this.env= env;
this.jobLogs = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs"); this.jobLogs = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
@ -50,7 +50,7 @@ public class ProductionPublishingJob extends DMPMJob {
***REMOVED***if (CheckPermission.apply(targetVREToken,targetVREName)){ ***REMOVED***if (CheckPermission.apply(targetVREToken,targetVREName)){
***REMOVED***this.svnUpdater.updateProdDeps(this.algorithm); ***REMOVED***this.svnUpdater.updateProdDeps(this.algorithm);
this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.targetVREName, this.category,this.algorithm_type, this.algorithm.getFullname(), env); this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, this.category,this.algorithm_type, this.algorithm.getFullname()/*, env*/);
this.getStatus(9); this.getStatus(9);
sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBodyRelease("\n\n"+this.buildInfo())); sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBodyRelease("\n\n"+this.buildInfo()));
return; return;

22
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
View File

@ -27,14 +27,14 @@ public class StagingJob extends DMPMJob {
private Cluster stagingCluster; private Cluster stagingCluster;
***REMOVED*** private Cluster rProtoCluster; ***REMOVED*** private Cluster rProtoCluster;
private String rProtoVREName; private String rProtoVREName;
private String env; ***REMOVED***private String env;
private String category; private String category;
private String algorithm_type; private String algorithm_type;
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm,
Cluster stagingCluster, /* Cluster rProtoCluster, */ Cluster stagingCluster, /* Cluster rProtoCluster, */
String rProtoVREName, String category, String algorithm_type, String env) throws FileNotFoundException, UnsupportedEncodingException { String rProtoVREName, String category, String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException {
super(svnUpdater); super(svnUpdater);
this.jobLogs = new File( this.jobLogs = new File(
System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs"); System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
@ -44,7 +44,7 @@ public class StagingJob extends DMPMJob {
this.stagingCluster = stagingCluster; this.stagingCluster = stagingCluster;
***REMOVED*** this.rProtoCluster = rProtoCluster; ***REMOVED*** this.rProtoCluster = rProtoCluster;
this.rProtoVREName = rProtoVREName; this.rProtoVREName = rProtoVREName;
this.env = env; ***REMOVED***this.env = env;
this.category = category; this.category = category;
this.algorithm_type = algorithm_type; this.algorithm_type = algorithm_type;
@ -66,7 +66,7 @@ public class StagingJob extends DMPMJob {
try { try {
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies( Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
this.svnUpdater.getDependencyFile(this.algorithm.getLanguage(),env), this.svnUpdater.getDependencyFile(this.algorithm.getLanguage()/*,env*/),
this.algorithm.getDependencies()); this.algorithm.getDependencies());
if (!undefinedDependencies.isEmpty()) { if (!undefinedDependencies.isEmpty()) {
@ -82,7 +82,7 @@ public class StagingJob extends DMPMJob {
***REMOVED*** ***REMOVED***
***REMOVED***before the installation to check if the files exist ***REMOVED***before the installation to check if the files exist
b.deleteFiles(this.algorithm, env);; b.deleteFiles(this.algorithm/*, env*/);;
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root"));
@ -95,18 +95,18 @@ public class StagingJob extends DMPMJob {
if (ret == 0) { if (ret == 0) {
this.getStatus(0); this.getStatus(0);
System.out.println("1 - Checking existing in env: "+ env); ***REMOVED***System.out.println("1 - Checking existing in env: "+ env);
System.out.println("2 - Checking existing in env: "+ this.env); ***REMOVED***System.out.println("2 - Checking existing in env: "+ this.env);
if (b.checkMethod(a.getHost(env), SecurityTokenProvider.instance.get())&&(b.algoExists(this.algorithm, env))) { if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get())&&(b.algoExists(this.algorithm/*, env*/))) {
System.out.println("Interface check ok!"); System.out.println("Interface check ok!");
System.out.println("Both the files exist at the correct path!"); System.out.println("Both the files exist at the correct path!");
this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.rProtoVREName,this.category, this.algorithm_type, this.svnUpdater.updateSVNStagingAlgorithmList(this.algorithm, this.rProtoVREName,this.category, this.algorithm_type,
this.algorithm.getFullname(), env); this.algorithm.getFullname()/*, env*/);
this.getStatus(9); this.getStatus(9);
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo())); sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
@ -163,7 +163,7 @@ public class StagingJob extends DMPMJob {
"Algorithm details:\n"+"\n"+ "Algorithm details:\n"+"\n"+
"User: "+this.algorithm.getFullname()+"\n"+ "User: "+this.algorithm.getFullname()+"\n"+
"Algorithm name: "+this.algorithm.getName()+"\n"+ "Algorithm name: "+this.algorithm.getName()+"\n"+
"Staging DataMiner Host: "+ a.getHost(this.env)+"\n"+ "Staging DataMiner Host: "+ a.getStagingHost()+"\n"+
"Caller VRE: "+ScopeProvider.instance.get()+"\n"+ "Caller VRE: "+ScopeProvider.instance.get()+"\n"+
"Target VRE: "+rProtoVREName+"\n"; "Target VRE: "+rProtoVREName+"\n";
***REMOVED*** ***REMOVED***

27
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java
View File

@ -84,8 +84,7 @@ public class CheckMethod {
public boolean algoExists(Algorithm a, String env) throws Exception{ public boolean algoExists(Algorithm a/*, String env*/) throws Exception{
System.out.println("checking existing in env: "+ env);
File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar"); File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar");
File file2 = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+"_interface.jar"); File file2 = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+"_interface.jar");
@ -95,9 +94,9 @@ public class CheckMethod {
System.out.println("Second file is located to: "+file2.getPath()); System.out.println("Second file is located to: "+file2.getPath());
if ((this.doesExist(file.getPath(),env)) && (this.doesExist(file2.getPath(),env))){ if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/))){
this.copyFromDmToSVN(file,env); this.copyFromDmToSVN(file/*,env*/);
this.copyFromDmToSVN(file2,env); this.copyFromDmToSVN(file2/*,env*/);
return true; return true;
@ -108,13 +107,13 @@ public class CheckMethod {
***REMOVED*** ***REMOVED***
public void deleteFiles(Algorithm a,String env) throws Exception{ public void deleteFiles(Algorithm a/*,String env*/) throws Exception{
JSch jsch = new JSch(); JSch jsch = new JSch();
Session session = null; Session session = null;
Channel channel = null; Channel channel = null;
ChannelSftp c = null; ChannelSftp c = null;
ServiceConfiguration p = new ServiceConfiguration(); ServiceConfiguration p = new ServiceConfiguration();
System.out.println("checking existing in env: "+ env + " " + p.getHost(env)); System.out.println("checking existing in env: " + p.getStagingHost());
File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar"); File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar");
File file2 = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+"_interface.jar"); File file2 = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+"_interface.jar");
@ -130,7 +129,7 @@ public class CheckMethod {
jsch.addIdentity(privateKey); jsch.addIdentity(privateKey);
System.out.println("Private Key Added."); System.out.println("Private Key Added.");
session = jsch.getSession("root", p.getHost(env)); session = jsch.getSession("root", p.getStagingHost());
System.out.println("session created."); System.out.println("session created.");
java.util.Properties config = new java.util.Properties(); java.util.Properties config = new java.util.Properties();
@ -145,7 +144,7 @@ public class CheckMethod {
c = (ChannelSftp) channel; c = (ChannelSftp) channel;
if(doesExist(file.getPath(),env)&&(doesExist(file2.getPath(),env))){ if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){
c.rm(file.getPath()); c.rm(file.getPath());
c.rm(file2.getPath()); c.rm(file2.getPath());
@ -163,7 +162,7 @@ public class CheckMethod {
public boolean doesExist(String path, String env) throws Exception { public boolean doesExist(String path/*, String env*/) throws Exception {
JSch jsch = new JSch(); JSch jsch = new JSch();
Session session = null; Session session = null;
Channel channel = null; Channel channel = null;
@ -178,7 +177,7 @@ public class CheckMethod {
jsch.addIdentity(privateKey); jsch.addIdentity(privateKey);
System.out.println("Private Key Added."); System.out.println("Private Key Added.");
session = jsch.getSession("root", p.getHost(env)); session = jsch.getSession("root", p.getStagingHost());
System.out.println("session created."); System.out.println("session created.");
java.util.Properties config = new java.util.Properties(); java.util.Properties config = new java.util.Properties();
@ -217,7 +216,7 @@ public class CheckMethod {
public void copyFromDmToSVN(File a,String env) throws Exception { public void copyFromDmToSVN(File a/*,String env*/) throws Exception {
JSch jsch = new JSch(); JSch jsch = new JSch();
Session session = null; Session session = null;
ServiceConfiguration sc = new ServiceConfiguration(); ServiceConfiguration sc = new ServiceConfiguration();
@ -231,7 +230,7 @@ public class CheckMethod {
jsch.addIdentity(privateKey); jsch.addIdentity(privateKey);
System.out.println("Private Key Added."); System.out.println("Private Key Added.");
session = jsch.getSession("root", p.getHost(env)); session = jsch.getSession("root", p.getStagingHost());
System.out.println("session created."); System.out.println("session created.");
java.util.Properties config = new java.util.Properties(); java.util.Properties config = new java.util.Properties();
@ -304,7 +303,7 @@ public class CheckMethod {
Algorithm al = new Algorithm(); Algorithm al = new Algorithm();
al.setName("RBLACKBOX"); al.setName("RBLACKBOX");
a.deleteFiles(al, "Dev"); a.deleteFiles(al);

22
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
View File

@ -13,20 +13,22 @@ public class ClusterBuilder {
***REMOVED***1. to complete ***REMOVED***1. to complete
public static Cluster getStagingDataminerCluster(String env) throws FileNotFoundException{ public static Cluster getStagingDataminerCluster() throws FileNotFoundException{
Cluster cluster = new Cluster(); Cluster cluster = new Cluster();
ServiceConfiguration p = new ServiceConfiguration(); ServiceConfiguration p = new ServiceConfiguration();
Host h = new Host(); Host h = new Host();
h.setName(p.getStagingHost());
cluster.addHost(h);
if (env.equals("Dev")){ ***REMOVED*** if (env.equals("Dev")){
h.setName(p.getDevStagingHost()); ***REMOVED*** h.setName(p.getDevStagingHost());
cluster.addHost(h); ***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED***
if ((env.equals("Prod")||(env.equals("Proto")))){ ***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto")))){
h.setName(p.getProtoProdStagingHost()); ***REMOVED*** h.setName(p.getProtoProdStagingHost());
cluster.addHost(h); ***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED*** ***REMOVED***
return cluster; return cluster;
***REMOVED*** ***REMOVED***

471
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
View File

@ -50,98 +50,159 @@ public class SVNUpdater {
public String getDependencyFile(String language, String env){ public String getDependencyFile(String language/*, String env*/){
String a = ""; String a = "";
if (env.equals("Dev")){ ***REMOVED*** if (env.equals("Dev")){
a= this.getDevDependencyFile(language); ***REMOVED*** a= this.getDevDependencyFile(language);
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED***
if (env.equals("Prod")){ ***REMOVED*** if (env.equals("Prod")){
a= this.getProdDependencyFile(language); ***REMOVED*** a= this.getProdDependencyFile(language);
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED***
if (env.equals("Proto")){ ***REMOVED*** if (env.equals("Proto")){
a= this.getRProtoDependencyFile(language); ***REMOVED*** a= this.getRProtoDependencyFile(language);
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Preprod")){
***REMOVED*** a= this.getPreProdDependencyFile(language);
***REMOVED*** ***REMOVED***
a = this.getStagingDependencyFile(language);
return a; return a;
***REMOVED*** ***REMOVED***
public String getRProtoDependencyFile(String language) { ***REMOVED*** public String getRProtoDependencyFile(String language) {
switch (language) { ***REMOVED*** switch (language) {
case "R": ***REMOVED*** case "R":
return this.configuration.getSVNRProtoCRANDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
case "R-blackbox": ***REMOVED*** case "R-blackbox":
return this.configuration.getSVNRProtoRBDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoRBDepsList();
case "Java": ***REMOVED*** case "Java":
return this.configuration.getSVNRProtoJavaDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoJavaDepsList();
case "Knime-Workflow": ***REMOVED*** case "Knime-Workflow":
return this.configuration.getSVNRProtoKWDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoKWDepsList();
case "Linux-compiled": ***REMOVED*** case "Linux-compiled":
return this.configuration.getSVNRProtoLinuxCompiledDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoLinuxCompiledDepsList();
case "Octave": ***REMOVED*** case "Octave":
return this.configuration.getSVNRProtoOctaveDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoOctaveDepsList();
case "Python": ***REMOVED*** case "Python":
return this.configuration.getSVNRProtoPythonDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoPythonDepsList();
case "Pre-Installed": ***REMOVED*** case "Pre-Installed":
return this.configuration.getSVNRProtoPreInstalledDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoPreInstalledDepsList();
case "Windows-compiled": ***REMOVED*** case "Windows-compiled":
return this.configuration.getSVNRProtoWCDepsList(); ***REMOVED*** return this.configuration.getSVNRProtoWCDepsList();
default: ***REMOVED*** default:
return null; ***REMOVED*** return null;
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED******REMOVED***
public String getProdDependencyFile(String language) { ***REMOVED*** public String getPreProdDependencyFile(String language) {
switch (language) { ***REMOVED*** switch (language) {
case "R": ***REMOVED*** case "R":
return this.configuration.getSVNRProdCRANDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdCRANDepsList();
case "R-blackbox": ***REMOVED*** case "R-blackbox":
return this.configuration.getSVNRProdRBDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdRBDepsList();
case "Java": ***REMOVED*** case "Java":
return this.configuration.getSVNRProdJavaDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdJavaDepsList();
case "Knime-Workflow": ***REMOVED*** case "Knime-Workflow":
return this.configuration.getSVNRProdKWDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdKWDepsList();
case "Linux-compiled": ***REMOVED*** case "Linux-compiled":
return this.configuration.getSVNRProdLinuxCompiledDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdLinuxCompiledDepsList();
case "Octave": ***REMOVED*** case "Octave":
return this.configuration.getSVNRProdOctaveDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdOctaveDepsList();
case "Python": ***REMOVED*** case "Python":
return this.configuration.getSVNRProdPythonDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdPythonDepsList();
case "Pre-Installed": ***REMOVED*** case "Pre-Installed":
return this.configuration.getSVNRProdPreInstalledDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdPreInstalledDepsList();
case "Windows-compiled": ***REMOVED*** case "Windows-compiled":
return this.configuration.getSVNRProdWCDepsList(); ***REMOVED*** return this.configuration.getSVNPreProdWCDepsList();
default: ***REMOVED*** default:
return null; ***REMOVED*** return null;
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED******REMOVED***
***REMOVED*** public String getProdDependencyFile(String language) {
***REMOVED*** switch (language) {
***REMOVED*** case "R":
***REMOVED*** return this.configuration.getSVNProdCRANDepsList();
***REMOVED*** case "R-blackbox":
***REMOVED*** return this.configuration.getSVNProdRBDepsList();
***REMOVED*** case "Java":
***REMOVED*** return this.configuration.getSVNProdJavaDepsList();
***REMOVED*** case "Knime-Workflow":
***REMOVED*** return this.configuration.getSVNProdKWDepsList();
***REMOVED*** case "Linux-compiled":
***REMOVED*** return this.configuration.getSVNProdLinuxCompiledDepsList();
***REMOVED*** case "Octave":
***REMOVED*** return this.configuration.getSVNProdOctaveDepsList();
***REMOVED*** case "Python":
***REMOVED*** return this.configuration.getSVNProdPythonDepsList();
***REMOVED*** case "Pre-Installed":
***REMOVED*** return this.configuration.getSVNProdPreInstalledDepsList();
***REMOVED*** case "Windows-compiled":
***REMOVED*** return this.configuration.getSVNProdWCDepsList();
***REMOVED*** default:
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
public String getDevDependencyFile(String language) { ***REMOVED*** public String getDevDependencyFile(String language) {
***REMOVED*** switch (language) {
***REMOVED*** case "R":
***REMOVED*** return this.configuration.getSVNDevCRANDepsList();
***REMOVED*** case "R-blackbox":
***REMOVED*** return this.configuration.getSVNDevRBDepsList();
***REMOVED*** case "Java":
***REMOVED*** return this.configuration.getSVNDevJavaDepsList();
***REMOVED*** case "Knime-Workflow":
***REMOVED*** return this.configuration.getSVNDevKWDepsList();
***REMOVED*** case "Linux-compiled":
***REMOVED*** return this.configuration.getSVNDevLinuxCompiledDepsList();
***REMOVED*** case "Octave":
***REMOVED*** return this.configuration.getSVNDevOctaveDepsList();
***REMOVED*** case "Python":
***REMOVED*** return this.configuration.getSVNDevPythonDepsList();
***REMOVED*** case "Pre-Installed":
***REMOVED*** return this.configuration.getSVNDevPreInstalledDepsList();
***REMOVED*** case "Windows-compiled":
***REMOVED*** return this.configuration.getSVNDevWCDepsList();
***REMOVED*** default:
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
public String getStagingDependencyFile(String language) {
switch (language) { switch (language) {
case "R": case "R":
return this.configuration.getSVNRDevCRANDepsList(); return this.configuration.getSVNStagingCRANDepsList();
case "R-blackbox": case "R-blackbox":
return this.configuration.getSVNRDevRBDepsList(); return this.configuration.getSVNStagingRBDepsList();
case "Java": case "Java":
return this.configuration.getSVNRDevJavaDepsList(); return this.configuration.getSVNStagingJavaDepsList();
case "Knime-Workflow": case "Knime-Workflow":
return this.configuration.getSVNRDevKWDepsList(); return this.configuration.getSVNStagingKWDepsList();
case "Linux-compiled": case "Linux-compiled":
return this.configuration.getSVNRDevLinuxCompiledDepsList(); return this.configuration.getSVNStagingLinuxCompiledDepsList();
case "Octave": case "Octave":
return this.configuration.getSVNRDevOctaveDepsList(); return this.configuration.getSVNStagingOctaveDepsList();
case "Python": case "Python":
return this.configuration.getSVNRDevPythonDepsList(); return this.configuration.getSVNStagingPythonDepsList();
case "Pre-Installed": case "Pre-Installed":
return this.configuration.getSVNRDevPreInstalledDepsList(); return this.configuration.getSVNStagingPreInstalledDepsList();
case "Windows-compiled": case "Windows-compiled":
return this.configuration.getSVNRDevWCDepsList(); return this.configuration.getSVNStagingWCDepsList();
default: default:
return null; return null;
***REMOVED*** ***REMOVED***
@ -151,98 +212,129 @@ public class SVNUpdater {
***REMOVED*** public void readPreProdDeps(Algorithm algorithm) throws SVNException {
public void readRPRotoDeps(Algorithm algorithm) throws SVNException { ***REMOVED*** if (algorithm.getLanguage().equals("R")) {
if (algorithm.getLanguage().equals("R")) { ***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) {
if (algorithm.getLanguage().equals("R-blackbox")) { ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if (algorithm.getLanguage().equals("Java")) {
if (algorithm.getLanguage().equals("Java")) { ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) {
if (algorithm.getLanguage().equals("Knime-Workflow")) { ***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) {
if (algorithm.getLanguage().equals("Linux-compiled")) { ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if (algorithm.getLanguage().equals("Octave")) {
if (algorithm.getLanguage().equals("Octave")) { ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if (algorithm.getLanguage().equals("Python")) {
if (algorithm.getLanguage().equals("Python")) { ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) {
if (algorithm.getLanguage().equals("Windows-compiled")) { ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) {
if (algorithm.getLanguage().equals("Pre-Installed")) { ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED***
public void readProdDeps(Algorithm algorithm) throws SVNException {
if (algorithm.getLanguage().equals("R")) {
this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("R-blackbox")) {
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Java")) {
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Knime-Workflow")) {
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Linux-compiled")) {
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Octave")) {
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Python")) {
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Windows-compiled")) {
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Pre-Installed")) {
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public void readRPRotoDeps(Algorithm algorithm) throws SVNException {
***REMOVED*** if (algorithm.getLanguage().equals("R")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
public void readRDevDeps(Algorithm algorithm) throws SVNException { ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("R")) { ***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) {
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("R-blackbox")) { ***REMOVED*** if (algorithm.getLanguage().equals("Java")) {
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("Java")) { ***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) {
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("Knime-Workflow")) { ***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) {
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("Linux-compiled")) { ***REMOVED*** if (algorithm.getLanguage().equals("Octave")) {
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("Octave")) { ***REMOVED*** if (algorithm.getLanguage().equals("Python")) {
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("Python")) { ***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) {
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("Windows-compiled")) { ***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) {
this.checkIfAvaialable("", algorithm.getDependencies()); ***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED*** ***REMOVED***
if (algorithm.getLanguage().equals("Pre-Installed")) { ***REMOVED******REMOVED***
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public void readProdDeps(Algorithm algorithm) throws SVNException {
***REMOVED*** if (algorithm.getLanguage().equals("R")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNProdCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Java")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Octave")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Python")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public void readRDevDeps(Algorithm algorithm) throws SVNException {
***REMOVED*** if (algorithm.getLanguage().equals("R")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Java")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Octave")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Python")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
@ -255,23 +347,31 @@ public class SVNUpdater {
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies()); ***REMOVED*** this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
***REMOVED******REMOVED*** ***REMOVED******REMOVED***
public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user, String env) { ***REMOVED*** public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
***REMOVED***
if (env.equals("Dev")){ ***REMOVED*** if (env.equals("Dev")){
this.updateSVNAlgorithmList(this.configuration.getSVNDevAlgorithmsList(), algorithm, targetVRE, category,algorithm_type, user, env); ***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNDevAlgorithmsList(), algorithm, targetVRE, category,algorithm_type, user, env);
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED***
if (env.equals("Prod")){ ***REMOVED*** if (env.equals("Prod")){
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env); ***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED***
if (env.equals("Proto")){ ***REMOVED*** if (env.equals("Proto")){
this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env); ***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** if (env.equals("Preprod")){
***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNPreProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user, String env) { public void updateSVNStagingAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env); this.updateSVNAlgorithmList(this.configuration.getSVNStagingAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
***REMOVED***
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
***REMOVED*** ***REMOVED***
@ -377,7 +477,7 @@ public class SVNUpdater {
***REMOVED*** ***REMOVED***
public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user, String env) { public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
try { try {
System.out.println("Updating algorithm list: " + file); System.out.println("Updating algorithm list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@ -397,7 +497,7 @@ public class SVNUpdater {
***REMOVED*** ***REMOVED***
***REMOVED*** the algorithm is not in the list or must be overwritten cause some modification. Add it ***REMOVED*** the algorithm is not in the list or must be overwritten cause some modification. Add it
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, category,algorithm_type, env)); newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, category,algorithm_type/*, env*/));
***REMOVED*** Collections.sort(newContent); ***REMOVED*** Collections.sort(newContent);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
@ -432,7 +532,7 @@ public class SVNUpdater {
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type,String env) throws ParseException { public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException {
***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis()); ***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis());
***REMOVED***long unixTime = System.currentTimeMillis() / 1000L; ***REMOVED***long unixTime = System.currentTimeMillis() / 1000L;
@ -440,7 +540,7 @@ public class SVNUpdater {
sb.append(algorithm.getName() + " | "); sb.append(algorithm.getName() + " | ");
sb.append(algorithm.getFullname() + " | "); sb.append(algorithm.getFullname() + " | ");
sb.append(category + " | "); sb.append(category + " | ");
sb.append(env + " | "); ***REMOVED***sb.append(env + " | ");
sb.append("<notextile>./addAlgorithm.sh " + algorithm.getName() + " " + algorithm.getCategory() + " " sb.append("<notextile>./addAlgorithm.sh " + algorithm.getName() + " " + algorithm.getCategory() + " "
+ algorithm.getClazz() + " " + targetVRE + " " + algorithm_type + " N " + algorithm.getClazz() + " " + targetVRE + " " + algorithm_type + " N "
+ algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | "); + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
@ -454,6 +554,15 @@ public class SVNUpdater {
***REMOVED*** SendMail sm = new SendMail(); ***REMOVED*** SendMail sm = new SendMail();
***REMOVED*** NotificationHelper nh = new NotificationHelper(); ***REMOVED*** NotificationHelper nh = new NotificationHelper();
List<String> undefined = new LinkedList<String>();
***REMOVED***to fix in next release: if the file is not present for that language in the service.properties then skip and return null list of string
***REMOVED***just to uncomment the following lines
***REMOVED***if(file.isEmpty()){
***REMOVED***return undefined;
***REMOVED******REMOVED***
System.out.println("Checking dependencies list: " + file); System.out.println("Checking dependencies list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@ -464,10 +573,6 @@ public class SVNUpdater {
validDependencies.add(l.trim()); validDependencies.add(l.trim());
***REMOVED*** ***REMOVED***
List<String> undefined = new LinkedList<String>();
for(Dependency d: deps){ for(Dependency d: deps){
String depName = d.getName(); String depName = d.getName();
if(!validDependencies.contains(depName)){ if(!validDependencies.contains(depName)){

364
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java
View File

@ -48,45 +48,45 @@ public class ServiceConfiguration {
***REMOVED***RProto ***REMOVED***RProto
public String getSVNRProtoAlgorithmsList(){ ***REMOVED*** public String getSVNRProtoAlgorithmsList(){
return props.getProperty("svn.rproto.algorithms-list"); ***REMOVED*** return props.getProperty("svn.rproto.algorithms-list");
***REMOVED******REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNRProtoLinuxCompiledDepsList(){
public String getSVNRProtoLinuxCompiledDepsList(){ ***REMOVED*** return props.getProperty("svn.rproto.deps-linux-compiled");
return props.getProperty("svn.rproto.deps-linux-compiled"); ***REMOVED******REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNRProtoCRANDepsList(){
public String getSVNRProtoCRANDepsList(){ ***REMOVED*** return props.getProperty("svn.rproto.deps-r");
return props.getProperty("svn.rproto.deps-r"); ***REMOVED******REMOVED***
***REMOVED***
public String getSVNRProtoPreInstalledDepsList(){
return props.getProperty("svn.rproto.deps-pre-installed");
***REMOVED***
public String getSVNRProtoRBDepsList(){
return props.getProperty("svn.rproto.deps-r-blackbox");
***REMOVED***
public String getSVNRProtoJavaDepsList(){
return props.getProperty("svn.rproto.deps-java");
***REMOVED***
public String getSVNRProtoKWDepsList(){
return props.getProperty("svn.rproto.deps-knime-workflow");
***REMOVED***
public String getSVNRProtoOctaveDepsList(){
return props.getProperty("svn.rproto.deps-octave");
***REMOVED***
public String getSVNRProtoPythonDepsList(){
return props.getProperty("svn.rproto.deps-python");
***REMOVED***
public String getSVNRProtoWCDepsList(){
return props.getProperty("svn.rproto.deps-windows-compiled");
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNRProtoPreInstalledDepsList(){
***REMOVED*** return props.getProperty("svn.rproto.deps-pre-installed");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNRProtoRBDepsList(){
***REMOVED*** return props.getProperty("svn.rproto.deps-r-blackbox");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNRProtoJavaDepsList(){
***REMOVED*** return props.getProperty("svn.rproto.deps-java");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNRProtoKWDepsList(){
***REMOVED*** return props.getProperty("svn.rproto.deps-knime-workflow");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNRProtoOctaveDepsList(){
***REMOVED*** return props.getProperty("svn.rproto.deps-octave");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNRProtoPythonDepsList(){
***REMOVED*** return props.getProperty("svn.rproto.deps-python");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNRProtoWCDepsList(){
***REMOVED*** return props.getProperty("svn.rproto.deps-windows-compiled");
***REMOVED******REMOVED***
***REMOVED***Prod ***REMOVED***Prod
@ -94,131 +94,223 @@ public class ServiceConfiguration {
return props.getProperty("svn.prod.algorithms-list"); return props.getProperty("svn.prod.algorithms-list");
***REMOVED*** ***REMOVED***
public String getSVNRProdLinuxCompiledDepsList(){ ***REMOVED*** public String getSVNProdLinuxCompiledDepsList(){
return props.getProperty("svn.prod.deps-linux-compiled"); ***REMOVED*** return props.getProperty("svn.prod.deps-linux-compiled");
***REMOVED******REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNProdCRANDepsList(){
public String getSVNRProdCRANDepsList(){ ***REMOVED*** return props.getProperty("svn.prod.deps-r");
return props.getProperty("svn.prod.deps-r"); ***REMOVED******REMOVED***
***REMOVED***
public String getSVNRProdPreInstalledDepsList(){
return props.getProperty("svn.prod.deps-pre-installed");
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNProdPreInstalledDepsList(){
***REMOVED*** return props.getProperty("svn.prod.deps-pre-installed");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNProdRBDepsList(){
***REMOVED*** return props.getProperty("svn.prod.deps-r-blackbox");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNProdJavaDepsList(){
***REMOVED*** return props.getProperty("svn.prod.deps-java");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNProdKWDepsList(){
***REMOVED*** return props.getProperty("svn.prod.deps-knime-workflow");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNProdOctaveDepsList(){
***REMOVED*** return props.getProperty("svn.prod.deps-octave");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNProdPythonDepsList(){
***REMOVED*** return props.getProperty("svn.prod.deps-python");
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public String getSVNProdWCDepsList(){
***REMOVED*** return props.getProperty("svn.prod.deps-windows-compiled");
***REMOVED******REMOVED***
***REMOVED***
public String getSVNRProdRBDepsList(){
return props.getProperty("svn.prod.deps-r-blackbox");
***REMOVED***
public String getSVNRProdJavaDepsList(){ ***REMOVED***PreProd
return props.getProperty("svn.prod.deps-java"); ***REMOVED*** public String getSVNPreProdAlgorithmsList(){
***REMOVED*** return props.getProperty("svn.preprod.algorithms-list");
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNPreProdLinuxCompiledDepsList(){
public String getSVNRProdKWDepsList(){ ***REMOVED*** return props.getProperty("svn.preprod.deps-linux-compiled");
return props.getProperty("svn.prod.deps-knime-workflow"); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNPreProdCRANDepsList(){
public String getSVNRProdOctaveDepsList(){ ***REMOVED*** return props.getProperty("svn.preprod.deps-r");
return props.getProperty("svn.prod.deps-octave"); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNPreProdPreInstalledDepsList(){
public String getSVNRProdPythonDepsList(){ ***REMOVED*** return props.getProperty("svn.preprod.deps-pre-installed");
return props.getProperty("svn.prod.deps-python"); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNPreProdRBDepsList(){
public String getSVNRProdWCDepsList(){ ***REMOVED*** return props.getProperty("svn.preprod.deps-r-blackbox");
return props.getProperty("svn.prod.deps-windows-compiled"); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNPreProdJavaDepsList(){
***REMOVED*** return props.getProperty("svn.preprod.deps-java");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNPreProdKWDepsList(){
***REMOVED*** return props.getProperty("svn.preprod.deps-knime-workflow");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNPreProdOctaveDepsList(){
***REMOVED*** return props.getProperty("svn.preprod.deps-octave");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNPreProdPythonDepsList(){
***REMOVED*** return props.getProperty("svn.preprod.deps-python");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNPreProdWCDepsList(){
***REMOVED*** return props.getProperty("svn.preprod.deps-windows-compiled");
***REMOVED*** ***REMOVED***
***REMOVED***dev ***REMOVED***dev
public String getSVNDevAlgorithmsList(){ ***REMOVED*** public String getSVNDevAlgorithmsList(){
return props.getProperty("svn.dev.algorithms-list"); ***REMOVED*** return props.getProperty("svn.dev.algorithms-list");
***REMOVED*** ***REMOVED*** ***REMOVED***
public String getSVNRDevLinuxCompiledDepsList(){
return props.getProperty("svn.dev.deps-linux-compiled");
***REMOVED***
public String getSVNRDevCRANDepsList(){
return props.getProperty("svn.dev.deps-r");
***REMOVED***
public String getSVNRDevPreInstalledDepsList(){
return props.getProperty("svn.dev.deps-pre-installed");
***REMOVED***
public String getSVNRDevRBDepsList(){
return props.getProperty("svn.dev.deps-r-blackbox");
***REMOVED***
public String getSVNRDevJavaDepsList(){
return props.getProperty("svn.dev.deps-java");
***REMOVED***
public String getSVNRDevKWDepsList(){
return props.getProperty("svn.dev.deps-knime-workflow");
***REMOVED***
public String getSVNRDevOctaveDepsList(){
return props.getProperty("svn.dev.deps-octave");
***REMOVED***
public String getSVNRDevPythonDepsList(){
return props.getProperty("svn.dev.deps-python");
***REMOVED***
public String getSVNRDevWCDepsList(){
return props.getProperty("svn.dev.deps-windows-compiled");
***REMOVED***
public String getCSVUrl() {
return props.getProperty("HAPROXY_CSV");
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSVNDevLinuxCompiledDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-linux-compiled");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNDevCRANDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-r");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNDevPreInstalledDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-pre-installed");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNDevRBDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-r-blackbox");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNDevJavaDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-java");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNDevKWDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-knime-workflow");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNDevOctaveDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-octave");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNDevPythonDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-python");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getSVNDevWCDepsList(){
***REMOVED*** return props.getProperty("svn.dev.deps-windows-compiled");
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String getCSVUrl() {
***REMOVED*** return props.getProperty("HAPROXY_CSV");
***REMOVED******REMOVED***
public String getHost(String env){ ***REMOVED*** public String getHost(String env){
String a = ""; ***REMOVED*** String a = "";
***REMOVED***
if (env.equals("Dev")){ ***REMOVED*** if (env.equals("Dev")||(env.equals("Preprod"))){
a = this.getDevStagingHost().trim(); ***REMOVED*** a = this.getDevStagingHost().trim();
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED***
if (env.equals("Prod")||(env.equals("Prod"))){ ***REMOVED*** if (env.equals("Prod")||(env.equals("Proto"))){
a = this.getProtoProdStagingHost().trim(); ***REMOVED*** a = this.getProtoProdStagingHost().trim();
***REMOVED*** ***REMOVED*** ***REMOVED***
return a; ***REMOVED*** return a;
***REMOVED******REMOVED***
***REMOVED*** public String getDevStagingHost() {
***REMOVED*** return props.getProperty("DEV_STAGING_HOST");
***REMOVED******REMOVED***
public String getStagingHost() {
return props.getProperty("STAGE_GHOST");
***REMOVED*** ***REMOVED***
***REMOVED*** public String getProtoProdStagingHost() {
public String getDevStagingHost() { ***REMOVED*** return props.getProperty("PROTO_PROD_STAGING_HOST");
return props.getProperty("DEV_STAGING_HOST"); ***REMOVED******REMOVED***
***REMOVED***
***REMOVED***Staging
public String getSVNStagingAlgorithmsList(){
return props.getProperty("svn.stage.algorithms-list");
***REMOVED***
public String getSVNStagingLinuxCompiledDepsList(){
return props.getProperty("svn.stage.deps-linux-compiled");
***REMOVED***
public String getSVNStagingCRANDepsList(){
return props.getProperty("svn.stage.deps-r");
***REMOVED***
public String getSVNStagingPreInstalledDepsList(){
return props.getProperty("svn.stage.deps-pre-installed");
***REMOVED***
public String getSVNStagingRBDepsList(){
return props.getProperty("svn.stage.deps-r-blackbox");
***REMOVED***
public String getSVNStagingJavaDepsList(){
return props.getProperty("svn.stage.deps-java");
***REMOVED***
public String getSVNStagingKWDepsList(){
return props.getProperty("svn.stage.deps-knime-workflow");
***REMOVED***
public String getSVNStagingOctaveDepsList(){
return props.getProperty("svn.stage.deps-octave");
***REMOVED***
public String getSVNStagingPythonDepsList(){
return props.getProperty("svn.stage.deps-python");
***REMOVED***
public String getSVNStagingWCDepsList(){
return props.getProperty("svn.stage.deps-windows-compiled");
***REMOVED***
public String getProtoProdStagingHost() {
return props.getProperty("PROTO_PROD_STAGING_HOST");
***REMOVED***
public static void main(String[] args) throws FileNotFoundException { public static void main(String[] args) throws FileNotFoundException {
ServiceConfiguration a = new ServiceConfiguration(); ServiceConfiguration a = new ServiceConfiguration();
***REMOVED***System.out.println(a.getStagingHost()); ***REMOVED***System.out.println(a.getStagingHost());
System.out.println(a.getDevStagingHost()); ***REMOVED***System.out.println(a.getDevStagingHost());
System.out.println(a.getProtoProdStagingHost()); ***REMOVED***System.out.println(a.getProtoProdStagingHost());
System.out.println(a.getCSVUrl()); ***REMOVED***System.out.println(a.getCSVUrl());
System.out.println(a.getSVNMainAlgoRepo()); ***REMOVED***System.out.println(a.getSVNMainAlgoRepo());
System.out.println(a.getSVNRProtoCRANDepsList()); ***REMOVED***System.out.println(a.getSVNRProtoCRANDepsList());
System.out.println(a.getStagingHost());
***REMOVED*** ***REMOVED***

57
src/main/resources/service.properties
View File

@ -1,50 +1,23 @@
#YML node file #YML node file
DEV_STAGING_HOST: dataminer1-devnext.d4science.org SVN_REPO = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
PROTO_PROD_STAGING_HOST: dataminer-proto-ghost.d4science.org
SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
#HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0
svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube
svn.algo.main.repo = /trunk/data-analysis/DataMinerConfiguration/algorithms svn.algo.main.repo = /trunk/data-analysis/DataMinerConfiguration/algorithms
svn.rproto.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/proto/algorithms
svn.rproto.deps-linux-compiled = #STAGE
svn.rproto.deps-pre-installed = STAGE_GHOST = dataminer-ghost-d.dev.d4science.org
svn.rproto.deps-r-blackbox = svn.stage.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms
svn.rproto.deps-r = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_cran_pkgs.txt
svn.rproto.deps-java =
svn.rproto.deps-knime-workflow =
svn.rproto.deps-octave =
svn.rproto.deps-python =
svn.rproto.deps-windows-compiled =
svn.stage.deps-linux-compiled = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt
svn.stage.deps-pre-installed = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt
svn.stage.deps-r-blackbox = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
svn.stage.deps-r = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
svn.stage.deps-java =
svn.stage.deps-knime-workflow =
svn.stage.deps-octave =
svn.stage.deps-python =
svn.stage.deps-windows-compiled =
svn.prod.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/prod/algorithms #PROD
svn.prod.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/prod/algorithms
svn.prod.deps-linux-compiled =
svn.prod.deps-pre-installed =
svn.prod.deps-r-blackbox =
svn.prod.deps-r = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
svn.prod.deps-java =
svn.prod.deps-knime-workflow =
svn.prod.deps-octave =
svn.prod.deps-python =
svn.prod.deps-windows-compiled =
svn.dev.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms
svn.dev.deps-linux-compiled =
svn.dev.deps-pre-installed =
svn.dev.deps-r-blackbox =
svn.dev.deps-r = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
svn.dev.deps-java =
svn.dev.deps-knime-workflow =
svn.dev.deps-octave =
svn.dev.deps-python =
svn.dev.deps-windows-compiled =

6
src/main/webapp/WEB-INF/web.xml
View File

@ -5,12 +5,12 @@
<web-app> <web-app>
<display-name>Archetype Created Web Application</display-name> <display-name>Archetype Created Web Application</display-name>
<context-param> <!-- <context-param>
<param-name>Environment</param-name> <param-name>Environment</param-name>
<param-value>Dev</param-value> <param-value>Dev</param-value>
<!-- Possible Values: Dev, Proto, Prod --> Possible Values: Dev, Proto, Prod, Preprod
<description>This is a context parameter example</description> <description>This is a context parameter example</description>
</context-param> </context-param> -->
<servlet> <servlet>
<servlet-name>REST-API</servlet-name> <servlet-name>REST-API</servlet-name>