diff --git a/pom.xml b/pom.xml index 0198b42..e6dc196 100644 --- a/pom.xml +++ b/pom.xml @@ -1,169 +1,157 @@ - 4.0.0 + xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - - maven-parent - org.gcube.tools - 1.0.0 - - + + maven-parent + org.gcube.tools + 1.0.0 + + - org.gcube.dataAnalysis - dataminer-pool-manager - 2.0.0-SNAPSHOT - war + org.gcube.dataAnalysis + dataminer-pool-manager + 2.0.0-SNAPSHOT + war - dataminer-pool-manager - + dataminer-pool-manager + - - distro - UTF-8 - 1.5.13 - 2.25.1 - + + distro + UTF-8 + 1.5.13 + 2.25.1 + - + - - org.gcube.core - common-smartgears - provided - + + org.gcube.core + common-smartgears + provided + - - org.gcube.resources.discovery - ic-client - provided - + + org.gcube.resources.discovery + ic-client + provided + - - commons-io - commons-io - [2.5.0,2.6.0) - - - org.antlr - stringtemplate - [4.0.0, 4.1.0) - - - org.tmatesoft.svnkit - svnkit - 1.8.5 - - - commons-configuration - commons-configuration - 1.10 - - - - net.sf.opencsv - opencsv - 2.3 - - - - org.yaml - snakeyaml - 1.16 - - - org.glassfish.jersey.media - jersey-media-json-jackson - $***REMOVED***version.jersey***REMOVED*** - - - - org.glassfish.jersey.core - jersey-client - $***REMOVED***version.jersey***REMOVED*** - - - - org.glassfish.jersey.containers - jersey-container-servlet - $***REMOVED***version.jersey***REMOVED*** - - - - - org.slf4j - slf4j-api - provided - + + commons-io + commons-io + [2.5.0,2.6.0) + + + org.antlr + stringtemplate + [4.0.0, 4.1.0) + + + org.gcube.dataanalysis + ecological-engine-smart-executor + 1.4.0-SNAPSHOT + - - - - - org.gcube.distribution - maven-smartgears-bom - LATEST - pom - import - - - + + org.tmatesoft.svnkit + svnkit + 1.8.5 + + + commons-configuration + commons-configuration + 1.10 + - - + + net.sf.opencsv + opencsv + 2.3 + - - org.apache.maven.plugins - maven-assembly-plugin - - - $***REMOVED***distroDirectory***REMOVED***/descriptor.xml - - - - - servicearchive - install - - single - - - - + + org.yaml + snakeyaml + 1.16 + + + org.glassfish.jersey.media + jersey-media-json-jackson + $***REMOVED***version.jersey***REMOVED*** + - - + + org.glassfish.jersey.core + jersey-client + $***REMOVED***version.jersey***REMOVED*** + + + + org.glassfish.jersey.containers + jersey-container-servlet + $***REMOVED***version.jersey***REMOVED*** + + + + + org.slf4j + slf4j-api + provided + + + + + + + + + org.gcube.distribution + maven-smartgears-bom + LATEST + pom + import + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + $***REMOVED***distroDirectory***REMOVED***/descriptor.xml + + + + + servicearchive + install + + single + + + + + + + diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java index b4b616a..4dda6cd 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java @@ -9,8 +9,14 @@ import java.util.Vector; public class Algorithm ***REMOVED*** - - + private String username; + + private String fullname; + + private String email; + + private String language; + private String name; private String description; @@ -153,6 +159,42 @@ public class Algorithm ***REMOVED*** this.skipJava = skipJava; ***REMOVED*** +public String getUsername() ***REMOVED*** + return username; +***REMOVED*** + +public void setUsername(String username) ***REMOVED*** + this.username = username; +***REMOVED*** + +public String getFullname() ***REMOVED*** + return fullname; +***REMOVED*** + +public void setFullname(String fullname) ***REMOVED*** + this.fullname = fullname; +***REMOVED*** + +public String getEmail() ***REMOVED*** + return email; +***REMOVED*** + +public void setEmail(String email) ***REMOVED*** + this.email = email; +***REMOVED*** + +public String getLanguage() ***REMOVED*** + return language; +***REMOVED*** + +public void setLanguage(String language) ***REMOVED*** + this.language = language; +***REMOVED*** + +public void setActions(Collection actions) ***REMOVED*** + this.actions = actions; +***REMOVED*** + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java index 9f889d4..1f20c26 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java @@ -22,6 +22,16 @@ public class AlgorithmPackageParser ***REMOVED*** */ private static final String METADATA_FILE_NAME = "Info.txt"; + private static final String METADATA_USERNAME = "Username"; + + private static final String METADATA_FULLNAME = "Full Name"; + + private static final String METADATA_EMAIL = "Email"; + + private static final String METADATA_LANGUAGE = "Language"; + + private static final String METADATA_CATEGORY = "Algorithm Category"; + private static final String METADATA_ALGORITHM_NAME = "Algorithm Name"; private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description"; @@ -145,73 +155,93 @@ public class AlgorithmPackageParser ***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); - ***REMOVED***List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); + out.setEmail(extractSingleValue(metadata, METADATA_EMAIL)); + out.setFullname(extractSingleValue(metadata, METADATA_FULLNAME)); + out.setUsername(extractSingleValue(metadata, METADATA_USERNAME)); + out.setLanguage(extractSingleValue(metadata, METADATA_LANGUAGE)); + out.setCategory(extractSingleValue(metadata, METADATA_CATEGORY)); - - List rdependencies = extractMultipleValues(metadata, "cran"); - if (rdependencies != null) ***REMOVED*** - for (String pkg : rdependencies) ***REMOVED*** + List dependencies = extractMultipleValues(metadata, "Package Name"); + + if (dependencies != null) ***REMOVED*** + for (String pkg : dependencies) ***REMOVED*** Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("cran"); + dep.setName(pkg); out.addDependency(dep); ***REMOVED*** ***REMOVED*** - List defdependencies = extractMultipleValues(metadata, "Packages"); - if (defdependencies != null) ***REMOVED*** - for (String pkg : defdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - List osdependencies = extractMultipleValues(metadata, "os"); - if (osdependencies != null) ***REMOVED*** - for (String pkg : osdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - - - List gitdependencies = extractMultipleValues(metadata, "github"); - if (gitdependencies != null) ***REMOVED*** - for (String pkg : gitdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("github"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - List cdependencies = extractMultipleValues(metadata, "custom"); - if (cdependencies != null) ***REMOVED*** - for (String pkg : cdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("custom"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** + + + +***REMOVED*** List rdependencies = extractMultipleValues(metadata, "cran"); +***REMOVED*** if (rdependencies != null) ***REMOVED*** +***REMOVED*** for (String pkg : rdependencies) ***REMOVED*** +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("cran"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** List defdependencies = extractMultipleValues(metadata, "Packages"); +***REMOVED*** if (defdependencies != null) ***REMOVED*** +***REMOVED*** for (String pkg : defdependencies) ***REMOVED*** +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("os"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** List osdependencies = extractMultipleValues(metadata, "os"); +***REMOVED*** if (osdependencies != null) ***REMOVED*** +***REMOVED*** for (String pkg : osdependencies) ***REMOVED*** +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("os"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** List gitdependencies = extractMultipleValues(metadata, "github"); +***REMOVED*** if (gitdependencies != null) ***REMOVED*** +***REMOVED*** for (String pkg : gitdependencies) ***REMOVED*** +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("github"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** List cdependencies = extractMultipleValues(metadata, "custom"); +***REMOVED*** if (cdependencies != null) ***REMOVED*** +***REMOVED*** for (String pkg : cdependencies) ***REMOVED*** +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("custom"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** ***REMOVED*** if (pkg.startsWith("r:"))***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index 25c3812..eb14280 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -44,8 +44,8 @@ public interface PoolManager ***REMOVED*** String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException; - String stageAlgorithm(String algorithmPackageURL, String category) throws IOException, InterruptedException; - String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE, String category) throws IOException, InterruptedException; + String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException; + String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE) throws IOException, InterruptedException; Algorithm extractAlgorithm(String url) throws IOException; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java index 9559e99..55aada6 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -30,7 +30,7 @@ public class ProductionPublishingJob extends DMPMJob ***REMOVED*** this.svnUpdater.updateProdDeps(this.algorithm); - this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod"); + this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, this.algorithm.getFullname(), "Prod"); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index 707cbf2..fb2e7ab 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -36,25 +36,36 @@ public class StagingJob extends DMPMJob ***REMOVED*** protected void execute() ***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); CheckMethod b = new CheckMethod(); - - try ***REMOVED*** - - int ret = this.executeAnsibleWorker( - createWorker(this.algorithm, this.stagingCluster, true, "root")); - - if(ret == 0)***REMOVED*** - - if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get()))***REMOVED*** - - this.svnUpdater.updateRPRotoDeps(this.algorithm); - this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, "DataMiner Pool Manager", "Proto"); + try ***REMOVED*** + + if (this.svnUpdater.checkIfAvaialable(this.algorithm.getLanguage(), this.algorithm.getDependencies())) ***REMOVED*** + int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); -***REMOVED*** int ret2 = this.executeAnsibleWorker( -***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, false, "gcube")); + + + + if (ret == 0) ***REMOVED*** + + if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get()) + && (b.algoExists(this.algorithm))) ***REMOVED*** + + ***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm); + + this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, + this.algorithm.getFullname(), "Proto"); + + + + + + ***REMOVED*** int ret2 = this.executeAnsibleWorker( + ***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, + ***REMOVED*** false, "gcube")); + ***REMOVED*** ***REMOVED*** ***REMOVED*** - + ***REMOVED*** catch (Exception e) ***REMOVED*** e.printStackTrace(); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java index 00f95c1..7207b50 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java @@ -11,8 +11,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackagePars public class AlgorithmBuilder ***REMOVED*** - public static Algorithm create(String algorithmPackageURL, String category) throws IOException, InterruptedException ***REMOVED*** - return create(algorithmPackageURL, null, null, null, null, category, null, null); + public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException ***REMOVED*** + return create(algorithmPackageURL, null, null, null, null, null, null, null); ***REMOVED*** public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description, diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java index 9d6e329..286c643 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java @@ -1,10 +1,13 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import java.io.BufferedReader; +import java.io.File; import java.io.FileWriter; import java.io.InputStreamReader; ***REMOVED*** +***REMOVED*** + public class CheckMethod ***REMOVED*** public CheckMethod() ***REMOVED*** @@ -64,6 +67,20 @@ public class CheckMethod ***REMOVED*** ***REMOVED*** return true; ***REMOVED*** + + + + public boolean algoExists(Algorithm a)***REMOVED*** + File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar"); + File file2 = new File(" /home/gcube/wps_algorithms/algorithms/"+a.getName()+"interface_.jar"); + + if (file.exists()&&(file2.exists()))***REMOVED*** + return true; + ***REMOVED*** + else + System.out.println("Algorithm"+a.getName()+".jar"+ " and "+a.getName()+"interface_.jar files are not present at the expected path"); + return false; +***REMOVED*** public static void main(String[] args) throws Exception ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java new file mode 100644 index 0000000..354fd71 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -0,0 +1,98 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +import java.text.SimpleDateFormat; +import java.util.Calendar; + +import org.apache.commons.lang.StringUtils; + +import scala.actors.threadpool.Arrays; + +public class NotificationHelper ***REMOVED*** + + private Calendar startTime; + + private String taskId; + + private String scope; + + private Exception executionException; + + public NotificationHelper() ***REMOVED*** + ***REMOVED*** + + public void setStartTime(Calendar startTime) ***REMOVED*** + this.startTime = startTime; + ***REMOVED*** + + public void setTaskId(String taskId) ***REMOVED*** + this.taskId = taskId; + ***REMOVED*** + + public void setScope(String scope) ***REMOVED*** + this.scope = scope; + ***REMOVED*** + + private String getSubjectHeader() ***REMOVED*** + return "[[DataMinerGhostInstallationRequestReport]]"; + ***REMOVED*** + + private String getSpecificVREName() ***REMOVED*** + if(this.scope!=null) ***REMOVED*** + String[] parts = this.scope.split("/"); + if(parts.length>=3) ***REMOVED*** + return StringUtils.join(Arrays.copyOfRange(parts, 3, parts.length), "/"); + ***REMOVED*** + ***REMOVED*** + return this.scope; + ***REMOVED*** + + private boolean isError() ***REMOVED*** + return this.executionException!=null; + ***REMOVED*** + + public void setExecutionException(Exception executionException) ***REMOVED*** + this.executionException = executionException; + ***REMOVED*** + + private String getSuccessSubject() ***REMOVED*** + return String.format("%s Results for your experiment '%s' are ready", this.getSubjectHeader(), this.taskId); + ***REMOVED*** + + private String getFailedSubject() ***REMOVED*** + return String.format("%s An error occurred while executing your experiment '%s'", this.getSubjectHeader(), this.taskId); + ***REMOVED*** + + private String getFormattedStartTime() ***REMOVED*** + SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); + return sdf.format(this.startTime.getTime()); + ***REMOVED*** + + private String getSuccessBody() ***REMOVED*** + String message = String.format("Your experiment '%s' submitted on %s in the '%s' VRE completed successfully.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName()); + message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface."; + return message; + ***REMOVED*** + + private String getFailedBody() ***REMOVED*** + String message = String.format("An error occurred while executing your experiment '%s' submitted on %s in the '%s' VRE.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName()); + message+= "\n\nHere are the error details:\n\n" + this.executionException; + return message; + ***REMOVED*** + + public String getSubject() ***REMOVED*** + if(this.isError()) ***REMOVED*** + return this.getFailedSubject(); + ***REMOVED*** else ***REMOVED*** + return this.getSuccessSubject(); + ***REMOVED*** + ***REMOVED*** + + public String getBody() ***REMOVED*** + if(this.isError()) ***REMOVED*** + return this.getFailedBody(); + ***REMOVED*** else ***REMOVED*** + return this.getSuccessBody(); + ***REMOVED*** + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index 34d0d91..d52af35 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -40,6 +40,71 @@ public class SVNUpdater ***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); ***REMOVED*** + + + + public void readRPRotoDeps(Algorithm algorithm) throws SVNException***REMOVED*** + if (algorithm.getLanguage().equals("R"))***REMOVED*** + this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("R-blackbox"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Java"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Knime-Workflow"))***REMOVED*** + this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Linux-compiled"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Octave"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Python"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Windows-compiled"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Pre-Installed"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + ***REMOVED*** + + + public void readProdDeps(Algorithm algorithm) throws SVNException***REMOVED*** + if (algorithm.getLanguage().equals("R"))***REMOVED*** + this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("R-blackbox"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Java"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Knime-Workflow"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Linux-compiled"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Octave"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Python"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Windows-compiled"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Pre-Installed"))***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + ***REMOVED*** + + public void updateProdDeps(Algorithm algorithm)***REMOVED*** this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies()); @@ -74,7 +139,7 @@ public class SVNUpdater ***REMOVED*** ***REMOVED***the algorithm is not in the list. Add it List newContent = new LinkedList<>(Arrays.asList(lines)); - newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, user, env)); + newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env)); ***REMOVED***Collections.sort(newContent); @@ -111,10 +176,10 @@ public class SVNUpdater ***REMOVED*** ***REMOVED*** ***REMOVED*** - public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED*** + public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env)***REMOVED*** StringBuffer sb = new StringBuffer("| "); sb.append(algorithm.getName() +" | "); - sb.append(user + " | "); + sb.append(algorithm.getFullname() + " | "); sb.append(algorithm.getCategory() + " | "); sb.append(env + " | "); sb.append("./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); @@ -123,6 +188,50 @@ public class SVNUpdater ***REMOVED*** ***REMOVED*** + + public boolean checkIfAvaialable(String file, Collection deps) throws SVNException ***REMOVED*** + boolean check = false; + try***REMOVED*** + System.out.println("Checking dependencies list: " + file); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + + if(deps.isEmpty())***REMOVED*** + SendMail sm = new SendMail(); + NotificationHelper nh = new NotificationHelper(); + sm.sendNotification(nh.getSubject(), nh.getBody()); + ***REMOVED*** + + + else if (!deps.isEmpty()) ***REMOVED*** + List ldep = new LinkedList<>(); + for (Dependency d : deps) ***REMOVED*** + ldep.add(d.getName()); + ***REMOVED*** + for (String a : lines) ***REMOVED*** + for (String b : ldep) ***REMOVED*** + if (b.equals(a)) ***REMOVED*** + System.out.println("The following dependency is correctly written: "+b); + check = true; + ***REMOVED*** else + check = false; + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + ***REMOVED***catch (Exception a)***REMOVED*** + a.getMessage(); + ***REMOVED*** + + return check; + +***REMOVED*** + + + + + + public void updateSVN(String file, Collection deps) ***REMOVED*** try ***REMOVED*** System.out.println("Updating dependencies list: "+ file); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java new file mode 100644 index 0000000..3af7a94 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -0,0 +1,101 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +import java.net.URLEncoder; + +import org.apache.activemq.transport.stomp.Stomp.Headers.Send; +***REMOVED*** +***REMOVED*** +import org.gcube.contentmanagement.graphtools.utils.HttpRequest; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm; +import org.gcube.dataanalysis.executor.util.InfraRetrieval; + +public class SendMail extends StandardLocalInfraAlgorithm ***REMOVED*** + + public SendMail() ***REMOVED*** + ***REMOVED*** TODO Auto-generated constructor stub +***REMOVED*** + +public void sendNotification(String subject, String body) throws Exception ***REMOVED*** + + AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope "+ScopeProvider.instance.get()); + + String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); + + if (!serviceAddress.endsWith("/")) + serviceAddress = serviceAddress+"/"; + + String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" + SecurityTokenProvider.instance.get(); + requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error + + AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage); + + ***REMOVED*** put the sender, the recipients, subject and body of the mail here + subject=URLEncoder.encode(subject,"UTF-8"); + body=URLEncoder.encode(body,"UTF-8"); + String requestParameters = "sender=dataminer&recipients="+SecurityTokenProvider.instance.get()+"&subject="+subject+"&body="+body; + + String response = HttpRequest.sendPostRequest(requestForMessage, requestParameters); + AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK "); + + if (response==null)***REMOVED*** + Exception e = new Exception("Error in email sending response"); + throw e; + ***REMOVED*** +***REMOVED*** + +***REMOVED*** public void notifySubmitter(String a, String b) throws Exception ***REMOVED*** +***REMOVED*** NotificationHelper nh = new NotificationHelper(); +***REMOVED*** super.sendNotification(nh.getSubject(), +***REMOVED*** nh.getBody()); +***REMOVED*** ***REMOVED*** + + + +***REMOVED*** + public String getDescription() ***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** + + + +***REMOVED*** + public void init() throws Exception ***REMOVED*** +***REMOVED*** + +***REMOVED*** + + + +***REMOVED*** + protected void process() throws Exception ***REMOVED*** +***REMOVED*** + +***REMOVED*** + + + +***REMOVED*** + protected void setInputParameters() ***REMOVED*** +***REMOVED*** + +***REMOVED*** + + + +***REMOVED*** + public void shutdown() ***REMOVED*** +***REMOVED*** + +***REMOVED*** + + public static void main(String[] args) throws Exception ***REMOVED*** + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + SecurityTokenProvider.instance.set("***REMOVED***"); + + NotificationHelper nh = new NotificationHelper(); + SendMail sm = new SendMail(); + sm.sendNotification(nh.getSubject(), nh.getBody()); +***REMOVED*** +***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java index 096d8d1..a920f6b 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java @@ -21,7 +21,7 @@ public class JobTest ***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration()); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0"); ***REMOVED***test phase Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java index 1a5afda..35937ed 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -17,11 +17,12 @@ public class SVNTests ***REMOVED*** public static void main(String[] args) throws SVNException, IOException, InterruptedException ***REMOVED*** - SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ggiammat/tmp/dmpm.properties")); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); + SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ngalante/workspace/dataminer-pool-manager/src/main/resources/service.properties")); + + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0"); + ***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); - algo.setClazz(algo.getClazz() + "TEST"); - - svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); + ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); + svnUpdater.readRPRotoDeps(algo); ***REMOVED*** ***REMOVED***