This commit is contained in:
Nunzio Andrea Galante 2017-07-05 15:50:42 +00:00
parent d07cf6c2b4
commit 62088fcd1d
14 changed files with 650 additions and 245 deletions

282
pom.xml
View File

@ -1,169 +1,157 @@
<project xmlns="http:***REMOVED***maven.apache.org/POM/4.0.0" xmlns:xsi="http:***REMOVED***www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>maven-parent</artifactId>
<groupId>org.gcube.tools</groupId>
<version>1.0.0</version>
<relativePath/>
</parent>
<parent>
<artifactId>maven-parent</artifactId>
<groupId>org.gcube.tools</groupId>
<version>1.0.0</version>
<relativePath />
</parent>
<groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<version>2.0.0-SNAPSHOT</version>
<packaging>war</packaging>
<groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<version>2.0.0-SNAPSHOT</version>
<packaging>war</packaging>
<name>dataminer-pool-manager</name>
<description>
<name>dataminer-pool-manager</name>
<description>
</description>
<properties>
<distroDirectory>distro</distroDirectory>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<version.swagger>1.5.13</version.swagger>
<version.jersey>2.25.1</version.jersey>
</properties>
<properties>
<distroDirectory>distro</distroDirectory>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<version.swagger>1.5.13</version.swagger>
<version.jersey>2.25.1</version.jersey>
</properties>
<dependencies>
<dependencies>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
<artifactId>ic-client</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
<artifactId>ic-client</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>[2.5.0,2.6.0)</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>stringtemplate</artifactId>
<version>[4.0.0, 4.1.0)</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.5</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<!--
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-jersey2-jaxrs</artifactId>
<version>${version.swagger***REMOVED***</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
<version>${version.swagger***REMOVED***</version>
</dependency>
-->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>[2.5.0,2.6.0)</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>stringtemplate</artifactId>
<version>[4.0.0, 4.1.0)</version>
</dependency>
<dependency>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>ecological-engine-smart-executor</artifactId>
<version>1.4.0-SNAPSHOT</version>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-smartgears-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.5</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<build>
<plugins>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>${distroDirectory***REMOVED***/descriptor.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>servicearchive</id>
<phase>install</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
</plugins>
</build>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<!-- <dependency> <groupId>io.swagger</groupId> <artifactId>swagger-jersey2-jaxrs</artifactId>
<version>${version.swagger***REMOVED***</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId> <version>2.6.0</version> <scope>provided</scope>
</dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId> <version>2.6.0</version> <scope>provided</scope>
</dependency> <dependency> <groupId>io.swagger</groupId> <artifactId>swagger-annotations</artifactId>
<version>${version.swagger***REMOVED***</version> </dependency> -->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-smartgears-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>${distroDirectory***REMOVED***/descriptor.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>servicearchive</id>
<phase>install</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -9,8 +9,14 @@ import java.util.Vector;
public class Algorithm {
private String username;
private String fullname;
private String email;
private String language;
private String name;
private String description;
@ -153,6 +159,42 @@ public class Algorithm {
this.skipJava = skipJava;
***REMOVED***
public String getUsername() {
return username;
***REMOVED***
public void setUsername(String username) {
this.username = username;
***REMOVED***
public String getFullname() {
return fullname;
***REMOVED***
public void setFullname(String fullname) {
this.fullname = fullname;
***REMOVED***
public String getEmail() {
return email;
***REMOVED***
public void setEmail(String email) {
this.email = email;
***REMOVED***
public String getLanguage() {
return language;
***REMOVED***
public void setLanguage(String language) {
this.language = language;
***REMOVED***
public void setActions(Collection<Action> actions) {
this.actions = actions;
***REMOVED***
***REMOVED***

View File

@ -22,6 +22,16 @@ public class AlgorithmPackageParser {
*/
private static final String METADATA_FILE_NAME = "Info.txt";
private static final String METADATA_USERNAME = "Username";
private static final String METADATA_FULLNAME = "Full Name";
private static final String METADATA_EMAIL = "Email";
private static final String METADATA_LANGUAGE = "Language";
private static final String METADATA_CATEGORY = "Algorithm Category";
private static final String METADATA_ALGORITHM_NAME = "Algorithm Name";
private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description";
@ -145,73 +155,93 @@ public class AlgorithmPackageParser {
out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
***REMOVED***List<String> dependencies = extractMultipleValues(metadata, METADATA_PACKAGES);
out.setEmail(extractSingleValue(metadata, METADATA_EMAIL));
out.setFullname(extractSingleValue(metadata, METADATA_FULLNAME));
out.setUsername(extractSingleValue(metadata, METADATA_USERNAME));
out.setLanguage(extractSingleValue(metadata, METADATA_LANGUAGE));
out.setCategory(extractSingleValue(metadata, METADATA_CATEGORY));
List<String> rdependencies = extractMultipleValues(metadata, "cran");
if (rdependencies != null) {
for (String pkg : rdependencies) {
List<String> dependencies = extractMultipleValues(metadata, "Package Name");
if (dependencies != null) {
for (String pkg : dependencies) {
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("cran");
dep.setName(pkg);
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> defdependencies = extractMultipleValues(metadata, "Packages");
if (defdependencies != null) {
for (String pkg : defdependencies) {
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("os");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> osdependencies = extractMultipleValues(metadata, "os");
if (osdependencies != null) {
for (String pkg : osdependencies) {
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("os");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> gitdependencies = extractMultipleValues(metadata, "github");
if (gitdependencies != null) {
for (String pkg : gitdependencies) {
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("github");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> cdependencies = extractMultipleValues(metadata, "custom");
if (cdependencies != null) {
for (String pkg : cdependencies) {
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("custom");
out.addDependency(dep);
***REMOVED***
***REMOVED***
***REMOVED*** List<String> rdependencies = extractMultipleValues(metadata, "cran");
***REMOVED*** if (rdependencies != null) {
***REMOVED*** for (String pkg : rdependencies) {
***REMOVED*** Dependency dep = new Dependency();
***REMOVED***
***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("cran");
***REMOVED*** out.addDependency(dep);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** List<String> defdependencies = extractMultipleValues(metadata, "Packages");
***REMOVED*** if (defdependencies != null) {
***REMOVED*** for (String pkg : defdependencies) {
***REMOVED*** Dependency dep = new Dependency();
***REMOVED***
***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("os");
***REMOVED*** out.addDependency(dep);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** List<String> osdependencies = extractMultipleValues(metadata, "os");
***REMOVED*** if (osdependencies != null) {
***REMOVED*** for (String pkg : osdependencies) {
***REMOVED*** Dependency dep = new Dependency();
***REMOVED***
***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("os");
***REMOVED*** out.addDependency(dep);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** List<String> gitdependencies = extractMultipleValues(metadata, "github");
***REMOVED*** if (gitdependencies != null) {
***REMOVED*** for (String pkg : gitdependencies) {
***REMOVED*** Dependency dep = new Dependency();
***REMOVED***
***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("github");
***REMOVED*** out.addDependency(dep);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** List<String> cdependencies = extractMultipleValues(metadata, "custom");
***REMOVED*** if (cdependencies != null) {
***REMOVED*** for (String pkg : cdependencies) {
***REMOVED*** Dependency dep = new Dependency();
***REMOVED***
***REMOVED*** ***REMOVED***if (pkg.startsWith("os:")){
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("custom");
***REMOVED*** out.addDependency(dep);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** if (pkg.startsWith("r:")){

View File

@ -44,8 +44,8 @@ public interface PoolManager {
String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException;
String stageAlgorithm(String algorithmPackageURL, String category) throws IOException, InterruptedException;
String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE, String category) throws IOException, InterruptedException;
String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException;
String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE) throws IOException, InterruptedException;
Algorithm extractAlgorithm(String url) throws IOException;

View File

@ -37,9 +37,9 @@ public class RestPoolManager implements PoolManager {
@Path("/algorithm/stage")
@Produces("text/plain")
public String stageAlgorithm(
@QueryParam("algorithmPackageURL") String algorithmPackageURL,
@QueryParam("category") String category) throws IOException, InterruptedException {
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL, category);
@QueryParam("algorithmPackageURL") String algorithmPackageURL
/*@QueryParam("category") String category*/) throws IOException, InterruptedException {
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
return this.service.stageAlgorithm(algo);
***REMOVED***
@ -52,7 +52,7 @@ public class RestPoolManager implements PoolManager {
@QueryParam("targetVREToken") String targetVREToken,
@QueryParam("targetVRE") String targetVRE,
@QueryParam("category") String category) throws IOException, InterruptedException {
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL, category);
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
return this.service.publishAlgorithm(algo, targetVREToken, targetVRE);
***REMOVED***
@ -84,16 +84,16 @@ public class RestPoolManager implements PoolManager {
***REMOVED*** ***REMOVED*** ProxySelector.setDefault(new
***REMOVED*** ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
***REMOVED***
***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
***REMOVED*** SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
System.out.println(entry.getContext());
***REMOVED***AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
***REMOVED***System.out.println(entry.getContext());
***REMOVED***RestPoolManager a = new RestPoolManager();
RestPoolManager a = new RestPoolManager();
***REMOVED*** a.stageAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", category);
a.stageAlgorithm("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0");
***REMOVED*** ***REMOVED***a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext");
***REMOVED*** ***REMOVED*** PoolManager aa = new DataminerPoolManager();
***REMOVED***
@ -183,5 +183,13 @@ public class RestPoolManager implements PoolManager {
***REMOVED***
@Override
public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE)
throws IOException, InterruptedException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
***REMOVED***

View File

@ -30,7 +30,7 @@ public class ProductionPublishingJob extends DMPMJob {
this.svnUpdater.updateProdDeps(this.algorithm);
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, this.algorithm.getFullname(), "Prod");
***REMOVED***

View File

@ -36,25 +36,36 @@ public class StagingJob extends DMPMJob {
protected void execute() {
ServiceConfiguration a = new ServiceConfiguration();
CheckMethod b = new CheckMethod();
try {
int ret = this.executeAnsibleWorker(
createWorker(this.algorithm, this.stagingCluster, true, "root"));
if(ret == 0){
if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get())){
this.svnUpdater.updateRPRotoDeps(this.algorithm);
this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, "DataMiner Pool Manager", "Proto");
try {
if (this.svnUpdater.checkIfAvaialable(this.algorithm.getLanguage(), this.algorithm.getDependencies())) {
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root"));
***REMOVED*** int ret2 = this.executeAnsibleWorker(
***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
if (ret == 0) {
if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get())
&& (b.algoExists(this.algorithm))) {
***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm);
this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName,
this.algorithm.getFullname(), "Proto");
***REMOVED*** int ret2 = this.executeAnsibleWorker(
***REMOVED*** createWorker(this.algorithm, this.rProtoCluster,
***REMOVED*** false, "gcube"));
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** catch (Exception e) {
e.printStackTrace();
***REMOVED***

View File

@ -11,8 +11,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackagePars
public class AlgorithmBuilder {
public static Algorithm create(String algorithmPackageURL, String category) throws IOException, InterruptedException {
return create(algorithmPackageURL, null, null, null, null, category, null, null);
public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException {
return create(algorithmPackageURL, null, null, null, null, null, null, null);
***REMOVED***
public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description,

View File

@ -1,10 +1,13 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
***REMOVED***
import java.io.File;
import java.io.FileWriter;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
public class CheckMethod {
public CheckMethod() {
@ -64,6 +67,20 @@ public class CheckMethod {
***REMOVED***
return true;
***REMOVED***
public boolean algoExists(Algorithm a){
File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar");
File file2 = new File(" /home/gcube/wps_algorithms/algorithms/"+a.getName()+"interface_.jar");
if (file.exists()&&(file2.exists())){
return true;
***REMOVED***
else
System.out.println("Algorithm"+a.getName()+".jar"+ " and "+a.getName()+"interface_.jar files are not present at the expected path");
return false;
***REMOVED***
public static void main(String[] args) throws Exception {

View File

@ -0,0 +1,98 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import org.apache.commons.lang.StringUtils;
import scala.actors.threadpool.Arrays;
public class NotificationHelper {
private Calendar startTime;
private String taskId;
private String scope;
private Exception executionException;
public NotificationHelper() {
***REMOVED***
public void setStartTime(Calendar startTime) {
this.startTime = startTime;
***REMOVED***
public void setTaskId(String taskId) {
this.taskId = taskId;
***REMOVED***
public void setScope(String scope) {
this.scope = scope;
***REMOVED***
private String getSubjectHeader() {
return "[[DataMinerGhostInstallationRequestReport]]";
***REMOVED***
private String getSpecificVREName() {
if(this.scope!=null) {
String[] parts = this.scope.split("/");
if(parts.length>=3) {
return StringUtils.join(Arrays.copyOfRange(parts, 3, parts.length), "/");
***REMOVED***
***REMOVED***
return this.scope;
***REMOVED***
private boolean isError() {
return this.executionException!=null;
***REMOVED***
public void setExecutionException(Exception executionException) {
this.executionException = executionException;
***REMOVED***
private String getSuccessSubject() {
return String.format("%s Results for your experiment '%s' are ready", this.getSubjectHeader(), this.taskId);
***REMOVED***
private String getFailedSubject() {
return String.format("%s An error occurred while executing your experiment '%s'", this.getSubjectHeader(), this.taskId);
***REMOVED***
private String getFormattedStartTime() {
SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z");
return sdf.format(this.startTime.getTime());
***REMOVED***
private String getSuccessBody() {
String message = String.format("Your experiment '%s' submitted on %s in the '%s' VRE completed successfully.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName());
message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.";
return message;
***REMOVED***
private String getFailedBody() {
String message = String.format("An error occurred while executing your experiment '%s' submitted on %s in the '%s' VRE.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName());
message+= "\n\nHere are the error details:\n\n" + this.executionException;
return message;
***REMOVED***
public String getSubject() {
if(this.isError()) {
return this.getFailedSubject();
***REMOVED*** else {
return this.getSuccessSubject();
***REMOVED***
***REMOVED***
public String getBody() {
if(this.isError()) {
return this.getFailedBody();
***REMOVED*** else {
return this.getSuccessBody();
***REMOVED***
***REMOVED***
***REMOVED***

View File

@ -40,6 +40,71 @@ public class SVNUpdater {
this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies());
this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
***REMOVED***
public void readRPRotoDeps(Algorithm algorithm) throws SVNException{
if (algorithm.getLanguage().equals("R")){
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("R-blackbox")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Java")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Knime-Workflow")){
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Linux-compiled")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Octave")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Python")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Windows-compiled")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Pre-Installed")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
***REMOVED***
public void readProdDeps(Algorithm algorithm) throws SVNException{
if (algorithm.getLanguage().equals("R")){
this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("R-blackbox")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Java")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Knime-Workflow")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Linux-compiled")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Octave")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Python")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Windows-compiled")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
if (algorithm.getLanguage().equals("Pre-Installed")){
this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED***
***REMOVED***
public void updateProdDeps(Algorithm algorithm){
this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
@ -74,7 +139,7 @@ public class SVNUpdater {
***REMOVED***the algorithm is not in the list. Add it
List<String> newContent = new LinkedList<>(Arrays.asList(lines));
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, user, env));
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env));
***REMOVED***Collections.sort(newContent);
@ -111,10 +176,10 @@ public class SVNUpdater {
***REMOVED*** ***REMOVED***
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String user, String env){
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env){
StringBuffer sb = new StringBuffer("| ");
sb.append(algorithm.getName() +" | ");
sb.append(user + " | ");
sb.append(algorithm.getFullname() + " | ");
sb.append(algorithm.getCategory() + " | ");
sb.append(env + " | ");
sb.append("<notextile>./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
@ -123,6 +188,50 @@ public class SVNUpdater {
***REMOVED***
public boolean checkIfAvaialable(String file, Collection<Dependency> deps) throws SVNException {
boolean check = false;
try{
System.out.println("Checking dependencies list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
if(deps.isEmpty()){
SendMail sm = new SendMail();
NotificationHelper nh = new NotificationHelper();
sm.sendNotification(nh.getSubject(), nh.getBody());
***REMOVED***
else if (!deps.isEmpty()) {
List<String> ldep = new LinkedList<>();
for (Dependency d : deps) {
ldep.add(d.getName());
***REMOVED***
for (String a : lines) {
for (String b : ldep) {
if (b.equals(a)) {
System.out.println("The following dependency is correctly written: "+b);
check = true;
***REMOVED*** else
check = false;
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***catch (Exception a){
a.getMessage();
***REMOVED***
return check;
***REMOVED***
public void updateSVN(String file, Collection<Dependency> deps) {
try {
System.out.println("Updating dependencies list: "+ file);

View File

@ -0,0 +1,101 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.net.URLEncoder;
import org.apache.activemq.transport.stomp.Stomp.Headers.Send;
***REMOVED***
***REMOVED***
import org.gcube.contentmanagement.graphtools.utils.HttpRequest;
import org.gcube.contentmanagement.lexicalmatcher.utils.AnalysisLogger;
import org.gcube.dataanalysis.ecoengine.interfaces.StandardLocalInfraAlgorithm;
import org.gcube.dataanalysis.executor.util.InfraRetrieval;
public class SendMail extends StandardLocalInfraAlgorithm {
public SendMail() {
***REMOVED*** TODO Auto-generated constructor stub
***REMOVED***
public void sendNotification(String subject, String body) throws Exception {
AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope "+ScopeProvider.instance.get());
String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get());
if (!serviceAddress.endsWith("/"))
serviceAddress = serviceAddress+"/";
String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" + SecurityTokenProvider.instance.get();
requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error
AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage);
***REMOVED*** put the sender, the recipients, subject and body of the mail here
subject=URLEncoder.encode(subject,"UTF-8");
body=URLEncoder.encode(body,"UTF-8");
String requestParameters = "sender=dataminer&recipients="+SecurityTokenProvider.instance.get()+"&subject="+subject+"&body="+body;
String response = HttpRequest.sendPostRequest(requestForMessage, requestParameters);
AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK ");
if (response==null){
Exception e = new Exception("Error in email sending response");
throw e;
***REMOVED***
***REMOVED***
***REMOVED*** public void notifySubmitter(String a, String b) throws Exception {
***REMOVED*** NotificationHelper nh = new NotificationHelper();
***REMOVED*** super.sendNotification(nh.getSubject(),
***REMOVED*** nh.getBody());
***REMOVED*** ***REMOVED***
@Override
public String getDescription() {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
@Override
public void init() throws Exception {
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
@Override
protected void process() throws Exception {
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
@Override
protected void setInputParameters() {
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
@Override
public void shutdown() {
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
public static void main(String[] args) throws Exception {
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
NotificationHelper nh = new NotificationHelper();
SendMail sm = new SendMail();
sm.sendNotification(nh.getSubject(), nh.getBody());
***REMOVED***
***REMOVED***

View File

@ -21,7 +21,7 @@ public class JobTest {
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration());
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0");
***REMOVED***test phase
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();

View File

@ -17,11 +17,12 @@ public class SVNTests {
public static void main(String[] args) throws SVNException, IOException, InterruptedException {
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ggiammat/tmp/dmpm.properties"));
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ngalante/workspace/dataminer-pool-manager/src/main/resources/service.properties"));
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0");
***REMOVED***algo.setClazz(algo.getClazz() + "TEST");
algo.setClazz(algo.getClazz() + "TEST");
svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto");
***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto");
svnUpdater.readRPRotoDeps(algo);
***REMOVED***
***REMOVED***