diff --git a/pom.xml b/pom.xml index fd38a92..6f7a41f 100644 --- a/pom.xml +++ b/pom.xml @@ -1,169 +1,157 @@ - 4.0.0 + xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - - maven-parent - org.gcube.tools - 1.0.0 - - + + maven-parent + org.gcube.tools + 1.0.0 + + - org.gcube.dataAnalysis - dataminer-pool-manager - 2.0.0-SNAPSHOT - war + org.gcube.dataAnalysis + dataminer-pool-manager + 2.0.0-SNAPSHOT + war - dataminer-pool-manager - + dataminer-pool-manager + - - distro - UTF-8 - 1.5.13 - 2.25.1 - + + distro + UTF-8 + 1.5.13 + 2.25.1 + - + - - org.gcube.core - common-smartgears - provided - + + org.gcube.core + common-smartgears + provided + - - org.gcube.resources.discovery - ic-client - provided - + + org.gcube.resources.discovery + ic-client + provided + - - commons-io - commons-io - [2.5.0,2.6.0) - - - org.antlr - stringtemplate - [4.0.0, 4.1.0) - - - org.tmatesoft.svnkit - svnkit - 1.8.5 - - - commons-configuration - commons-configuration - 1.10 - - - - net.sf.opencsv - opencsv - 2.3 - - - - org.yaml - snakeyaml - 1.16 - - - org.glassfish.jersey.media - jersey-media-json-jackson - ${version.jersey***REMOVED*** - - - - org.glassfish.jersey.core - jersey-client - ${version.jersey***REMOVED*** - - - - org.glassfish.jersey.containers - jersey-container-servlet - ${version.jersey***REMOVED*** - - - - - org.slf4j - slf4j-api - provided - + + commons-io + commons-io + [2.5.0,2.6.0) + + + org.antlr + stringtemplate + [4.0.0, 4.1.0) + + + org.gcube.dataanalysis + ecological-engine-smart-executor + 1.4.0-SNAPSHOT + - - - - - org.gcube.distribution - maven-smartgears-bom - LATEST - pom - import - - - + + org.tmatesoft.svnkit + svnkit + 1.8.5 + + + commons-configuration + commons-configuration + 1.10 + - - + + net.sf.opencsv + opencsv + 2.3 + - - org.apache.maven.plugins - maven-assembly-plugin - - - ${distroDirectory***REMOVED***/descriptor.xml - - - - - servicearchive - install - - single - - - - + + org.yaml + snakeyaml + 1.16 + + + org.glassfish.jersey.media + jersey-media-json-jackson + ${version.jersey***REMOVED*** + - - + + org.glassfish.jersey.core + jersey-client + ${version.jersey***REMOVED*** + + + + org.glassfish.jersey.containers + jersey-container-servlet + ${version.jersey***REMOVED*** + + + + + org.slf4j + slf4j-api + provided + + + + + + + + + org.gcube.distribution + maven-smartgears-bom + LATEST + pom + import + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + ${distroDirectory***REMOVED***/descriptor.xml + + + + + servicearchive + install + + single + + + + + + + diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java index 0d942e2..2f5766d 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java @@ -9,8 +9,14 @@ import java.util.Vector; public class Algorithm { - - + private String username; + + private String fullname; + + private String email; + + private String language; + private String name; private String description; @@ -153,6 +159,42 @@ public class Algorithm { this.skipJava = skipJava; ***REMOVED*** +public String getUsername() { + return username; +***REMOVED*** + +public void setUsername(String username) { + this.username = username; +***REMOVED*** + +public String getFullname() { + return fullname; +***REMOVED*** + +public void setFullname(String fullname) { + this.fullname = fullname; +***REMOVED*** + +public String getEmail() { + return email; +***REMOVED*** + +public void setEmail(String email) { + this.email = email; +***REMOVED*** + +public String getLanguage() { + return language; +***REMOVED*** + +public void setLanguage(String language) { + this.language = language; +***REMOVED*** + +public void setActions(Collection actions) { + this.actions = actions; +***REMOVED*** + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java index 1af3d7c..1bf44e6 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java @@ -22,6 +22,16 @@ public class AlgorithmPackageParser { */ private static final String METADATA_FILE_NAME = "Info.txt"; + private static final String METADATA_USERNAME = "Username"; + + private static final String METADATA_FULLNAME = "Full Name"; + + private static final String METADATA_EMAIL = "Email"; + + private static final String METADATA_LANGUAGE = "Language"; + + private static final String METADATA_CATEGORY = "Algorithm Category"; + private static final String METADATA_ALGORITHM_NAME = "Algorithm Name"; private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description"; @@ -145,73 +155,93 @@ public class AlgorithmPackageParser { out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); - ***REMOVED***List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); + out.setEmail(extractSingleValue(metadata, METADATA_EMAIL)); + out.setFullname(extractSingleValue(metadata, METADATA_FULLNAME)); + out.setUsername(extractSingleValue(metadata, METADATA_USERNAME)); + out.setLanguage(extractSingleValue(metadata, METADATA_LANGUAGE)); + out.setCategory(extractSingleValue(metadata, METADATA_CATEGORY)); - - List rdependencies = extractMultipleValues(metadata, "cran"); - if (rdependencies != null) { - for (String pkg : rdependencies) { + List dependencies = extractMultipleValues(metadata, "Package Name"); + + if (dependencies != null) { + for (String pkg : dependencies) { Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("cran"); + dep.setName(pkg); out.addDependency(dep); ***REMOVED*** ***REMOVED*** - List defdependencies = extractMultipleValues(metadata, "Packages"); - if (defdependencies != null) { - for (String pkg : defdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - List osdependencies = extractMultipleValues(metadata, "os"); - if (osdependencies != null) { - for (String pkg : osdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - - - List gitdependencies = extractMultipleValues(metadata, "github"); - if (gitdependencies != null) { - for (String pkg : gitdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("github"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - List cdependencies = extractMultipleValues(metadata, "custom"); - if (cdependencies != null) { - for (String pkg : cdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("custom"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** + + + +***REMOVED*** List rdependencies = extractMultipleValues(metadata, "cran"); +***REMOVED*** if (rdependencies != null) { +***REMOVED*** for (String pkg : rdependencies) { +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("cran"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** List defdependencies = extractMultipleValues(metadata, "Packages"); +***REMOVED*** if (defdependencies != null) { +***REMOVED*** for (String pkg : defdependencies) { +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("os"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** List osdependencies = extractMultipleValues(metadata, "os"); +***REMOVED*** if (osdependencies != null) { +***REMOVED*** for (String pkg : osdependencies) { +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("os"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** List gitdependencies = extractMultipleValues(metadata, "github"); +***REMOVED*** if (gitdependencies != null) { +***REMOVED*** for (String pkg : gitdependencies) { +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("github"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** List cdependencies = extractMultipleValues(metadata, "custom"); +***REMOVED*** if (cdependencies != null) { +***REMOVED*** for (String pkg : cdependencies) { +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** +***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){ +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("custom"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** ***REMOVED*** if (pkg.startsWith("r:")){ diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index 2ab7fbf..dccb4f5 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -44,8 +44,8 @@ public interface PoolManager { String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException; - String stageAlgorithm(String algorithmPackageURL, String category) throws IOException, InterruptedException; - String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE, String category) throws IOException, InterruptedException; + String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException; + String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE) throws IOException, InterruptedException; Algorithm extractAlgorithm(String url) throws IOException; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index efb43e4..5851ef0 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -37,9 +37,9 @@ public class RestPoolManager implements PoolManager { @Path("/algorithm/stage") @Produces("text/plain") public String stageAlgorithm( - @QueryParam("algorithmPackageURL") String algorithmPackageURL, - @QueryParam("category") String category) throws IOException, InterruptedException { - Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL, category); + @QueryParam("algorithmPackageURL") String algorithmPackageURL + /*@QueryParam("category") String category*/) throws IOException, InterruptedException { + Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); return this.service.stageAlgorithm(algo); ***REMOVED*** @@ -52,7 +52,7 @@ public class RestPoolManager implements PoolManager { @QueryParam("targetVREToken") String targetVREToken, @QueryParam("targetVRE") String targetVRE, @QueryParam("category") String category) throws IOException, InterruptedException { - Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL, category); + Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); return this.service.publishAlgorithm(algo, targetVREToken, targetVRE); ***REMOVED*** @@ -84,16 +84,16 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** ***REMOVED*** ProxySelector.setDefault(new ***REMOVED*** ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); ***REMOVED*** -***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); - SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); +***REMOVED*** SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); - AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); - System.out.println(entry.getContext()); + ***REMOVED***AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); + ***REMOVED***System.out.println(entry.getContext()); - ***REMOVED***RestPoolManager a = new RestPoolManager(); + RestPoolManager a = new RestPoolManager(); -***REMOVED*** a.stageAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", category); + a.stageAlgorithm("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0"); ***REMOVED*** ***REMOVED***a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext"); ***REMOVED*** ***REMOVED*** PoolManager aa = new DataminerPoolManager(); ***REMOVED*** @@ -183,5 +183,13 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** + @Override + public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE) + throws IOException, InterruptedException { + ***REMOVED*** TODO Auto-generated method stub + return null; +***REMOVED*** + + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java index ac1fd34..683dc46 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -30,7 +30,7 @@ public class ProductionPublishingJob extends DMPMJob { this.svnUpdater.updateProdDeps(this.algorithm); - this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod"); + this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, this.algorithm.getFullname(), "Prod"); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index 7e6ce40..91ce468 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -36,25 +36,36 @@ public class StagingJob extends DMPMJob { protected void execute() { ServiceConfiguration a = new ServiceConfiguration(); CheckMethod b = new CheckMethod(); - - try { - - int ret = this.executeAnsibleWorker( - createWorker(this.algorithm, this.stagingCluster, true, "root")); - - if(ret == 0){ - - if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get())){ - - this.svnUpdater.updateRPRotoDeps(this.algorithm); - this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, "DataMiner Pool Manager", "Proto"); + try { + + if (this.svnUpdater.checkIfAvaialable(this.algorithm.getLanguage(), this.algorithm.getDependencies())) { + int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); -***REMOVED*** int ret2 = this.executeAnsibleWorker( -***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, false, "gcube")); + + + + if (ret == 0) { + + if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get()) + && (b.algoExists(this.algorithm))) { + + ***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm); + + this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, + this.algorithm.getFullname(), "Proto"); + + + + + + ***REMOVED*** int ret2 = this.executeAnsibleWorker( + ***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, + ***REMOVED*** false, "gcube")); + ***REMOVED*** ***REMOVED*** ***REMOVED*** - + ***REMOVED*** catch (Exception e) { e.printStackTrace(); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java index 70af35f..b55a520 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java @@ -11,8 +11,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackagePars public class AlgorithmBuilder { - public static Algorithm create(String algorithmPackageURL, String category) throws IOException, InterruptedException { - return create(algorithmPackageURL, null, null, null, null, category, null, null); + public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException { + return create(algorithmPackageURL, null, null, null, null, null, null, null); ***REMOVED*** public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description, diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java index 726242d..94d6de2 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java @@ -1,10 +1,13 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; ***REMOVED*** +import java.io.File; import java.io.FileWriter; ***REMOVED*** ***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; + public class CheckMethod { public CheckMethod() { @@ -64,6 +67,20 @@ public class CheckMethod { ***REMOVED*** return true; ***REMOVED*** + + + + public boolean algoExists(Algorithm a){ + File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar"); + File file2 = new File(" /home/gcube/wps_algorithms/algorithms/"+a.getName()+"interface_.jar"); + + if (file.exists()&&(file2.exists())){ + return true; + ***REMOVED*** + else + System.out.println("Algorithm"+a.getName()+".jar"+ " and "+a.getName()+"interface_.jar files are not present at the expected path"); + return false; +***REMOVED*** public static void main(String[] args) throws Exception { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java new file mode 100644 index 0000000..1925509 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -0,0 +1,98 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +import java.text.SimpleDateFormat; +import java.util.Calendar; + +import org.apache.commons.lang.StringUtils; + +import scala.actors.threadpool.Arrays; + +public class NotificationHelper { + + private Calendar startTime; + + private String taskId; + + private String scope; + + private Exception executionException; + + public NotificationHelper() { + ***REMOVED*** + + public void setStartTime(Calendar startTime) { + this.startTime = startTime; + ***REMOVED*** + + public void setTaskId(String taskId) { + this.taskId = taskId; + ***REMOVED*** + + public void setScope(String scope) { + this.scope = scope; + ***REMOVED*** + + private String getSubjectHeader() { + return "[[DataMinerGhostInstallationRequestReport]]"; + ***REMOVED*** + + private String getSpecificVREName() { + if(this.scope!=null) { + String[] parts = this.scope.split("/"); + if(parts.length>=3) { + return StringUtils.join(Arrays.copyOfRange(parts, 3, parts.length), "/"); + ***REMOVED*** + ***REMOVED*** + return this.scope; + ***REMOVED*** + + private boolean isError() { + return this.executionException!=null; + ***REMOVED*** + + public void setExecutionException(Exception executionException) { + this.executionException = executionException; + ***REMOVED*** + + private String getSuccessSubject() { + return String.format("%s Results for your experiment '%s' are ready", this.getSubjectHeader(), this.taskId); + ***REMOVED*** + + private String getFailedSubject() { + return String.format("%s An error occurred while executing your experiment '%s'", this.getSubjectHeader(), this.taskId); + ***REMOVED*** + + private String getFormattedStartTime() { + SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); + return sdf.format(this.startTime.getTime()); + ***REMOVED*** + + private String getSuccessBody() { + String message = String.format("Your experiment '%s' submitted on %s in the '%s' VRE completed successfully.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName()); + message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface."; + return message; + ***REMOVED*** + + private String getFailedBody() { + String message = String.format("An error occurred while executing your experiment '%s' submitted on %s in the '%s' VRE.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName()); + message+= "\n\nHere are the error details:\n\n" + this.executionException; + return message; + ***REMOVED*** + + public String getSubject() { + if(this.isError()) { + return this.getFailedSubject(); + ***REMOVED*** else { + return this.getSuccessSubject(); + ***REMOVED*** + ***REMOVED*** + + public String getBody() { + if(this.isError()) { + return this.getFailedBody(); + ***REMOVED*** else { + return this.getSuccessBody(); + ***REMOVED*** + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index 9cc200b..f6925a2 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -40,6 +40,71 @@ public class SVNUpdater { this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); ***REMOVED*** + + + + public void readRPRotoDeps(Algorithm algorithm) throws SVNException{ + if (algorithm.getLanguage().equals("R")){ + this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("R-blackbox")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Java")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Knime-Workflow")){ + this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Linux-compiled")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Octave")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Python")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Windows-compiled")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Pre-Installed")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + ***REMOVED*** + + + public void readProdDeps(Algorithm algorithm) throws SVNException{ + if (algorithm.getLanguage().equals("R")){ + this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("R-blackbox")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Java")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Knime-Workflow")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Linux-compiled")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Octave")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Python")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Windows-compiled")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Pre-Installed")){ + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + ***REMOVED*** + + public void updateProdDeps(Algorithm algorithm){ this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies()); @@ -74,7 +139,7 @@ public class SVNUpdater { ***REMOVED***the algorithm is not in the list. Add it List newContent = new LinkedList<>(Arrays.asList(lines)); - newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, user, env)); + newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env)); ***REMOVED***Collections.sort(newContent); @@ -111,10 +176,10 @@ public class SVNUpdater { ***REMOVED*** ***REMOVED*** - public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String user, String env){ + public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env){ StringBuffer sb = new StringBuffer("| "); sb.append(algorithm.getName() +" | "); - sb.append(user + " | "); + sb.append(algorithm.getFullname() + " | "); sb.append(algorithm.getCategory() + " | "); sb.append(env + " | "); sb.append("./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); @@ -123,6 +188,50 @@ public class SVNUpdater { ***REMOVED*** + + public boolean checkIfAvaialable(String file, Collection deps) throws SVNException { + boolean check = false; + try{ + System.out.println("Checking dependencies list: " + file); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + + if(deps.isEmpty()){ + SendMail sm = new SendMail(); + NotificationHelper nh = new NotificationHelper(); + sm.sendNotification(nh.getSubject(), nh.getBody()); + ***REMOVED*** + + + else if (!deps.isEmpty()) { + List ldep = new LinkedList<>(); + for (Dependency d : deps) { + ldep.add(d.getName()); + ***REMOVED*** + for (String a : lines) { + for (String b : ldep) { + if (b.equals(a)) { + System.out.println("The following dependency is correctly written: "+b); + check = true; + ***REMOVED*** else + check = false; + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + ***REMOVED***catch (Exception a){ + a.getMessage(); + ***REMOVED*** + + return check; + +***REMOVED*** + + + + + + public void updateSVN(String file, Collection deps) { try { System.out.println("Updating dependencies list: "+ file); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java new file mode 100644 index 0000000..2d82840 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -0,0 +1,101 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +import java.net.URLEncoder; + +import org.apache.activemq.transport.stomp.Stomp.Headers.Send; +***REMOVED*** +***REMOVED*** +import org.gcube.contentmanagement.graphtools.utils.HttpRequest; +import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger; +import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm; +import org.gcube.dataanalysis.executor.util.InfraRetrieval; + +public class SendMail extends StandardLocalInfraAlgorithm { + + public SendMail() { + ***REMOVED*** TODO Auto-generated constructor stub +***REMOVED*** + +public void sendNotification(String subject, String body) throws Exception { + + AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope "+ScopeProvider.instance.get()); + + String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); + + if (!serviceAddress.endsWith("/")) + serviceAddress = serviceAddress+"/"; + + String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" + SecurityTokenProvider.instance.get(); + requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error + + AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage); + + ***REMOVED*** put the sender, the recipients, subject and body of the mail here + subject=URLEncoder.encode(subject,"UTF-8"); + body=URLEncoder.encode(body,"UTF-8"); + String requestParameters = "sender=dataminer&recipients="+SecurityTokenProvider.instance.get()+"&subject="+subject+"&body="+body; + + String response = HttpRequest.sendPostRequest(requestForMessage, requestParameters); + AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK "); + + if (response==null){ + Exception e = new Exception("Error in email sending response"); + throw e; + ***REMOVED*** +***REMOVED*** + +***REMOVED*** public void notifySubmitter(String a, String b) throws Exception { +***REMOVED*** NotificationHelper nh = new NotificationHelper(); +***REMOVED*** super.sendNotification(nh.getSubject(), +***REMOVED*** nh.getBody()); +***REMOVED*** ***REMOVED*** + + + + @Override + public String getDescription() { + ***REMOVED*** TODO Auto-generated method stub + return null; +***REMOVED*** + + + + @Override + public void init() throws Exception { + ***REMOVED*** TODO Auto-generated method stub + +***REMOVED*** + + + + @Override + protected void process() throws Exception { + ***REMOVED*** TODO Auto-generated method stub + +***REMOVED*** + + + + @Override + protected void setInputParameters() { + ***REMOVED*** TODO Auto-generated method stub + +***REMOVED*** + + + + @Override + public void shutdown() { + ***REMOVED*** TODO Auto-generated method stub + +***REMOVED*** + + public static void main(String[] args) throws Exception { + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); + + NotificationHelper nh = new NotificationHelper(); + SendMail sm = new SendMail(); + sm.sendNotification(nh.getSubject(), nh.getBody()); +***REMOVED*** +***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java index 3653eb7..5350f60 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java @@ -21,7 +21,7 @@ public class JobTest { ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration()); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0"); ***REMOVED***test phase Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java index 53122cb..064d356 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -17,11 +17,12 @@ public class SVNTests { public static void main(String[] args) throws SVNException, IOException, InterruptedException { - SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ggiammat/tmp/dmpm.properties")); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); + SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ngalante/workspace/dataminer-pool-manager/src/main/resources/service.properties")); + + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0"); + ***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); - algo.setClazz(algo.getClazz() + "TEST"); - - svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); + ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); + svnUpdater.readRPRotoDeps(algo); ***REMOVED*** ***REMOVED***