diff --git a/pom.xml b/pom.xml
index 042fb14..8a14623 100644
--- a/pom.xml
+++ b/pom.xml
@@ -12,7 +12,7 @@
org.gcube.dataAnalysis
dataminer-pool-manager
war
- 2.2.0-SNAPSHOT
+ 2.3.0-SNAPSHOT
dataminer-pool-manager
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
index cae5f31..885736b 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
@@ -1,41 +1,16 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible;
-***REMOVED***
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
import java.io.File;
***REMOVED***
import java.io.InputStream;
-***REMOVED***
import java.io.PrintStream;
-import java.io.PrintWriter;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-***REMOVED***
-***REMOVED***
import java.util.Scanner;
-import java.util.Set;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
-import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.tmatesoft.svn.core.SVNException;
-import org.tmatesoft.svn.core.SVNURL;
-import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
-import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
-import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
-import org.tmatesoft.svn.core.io.ISVNEditor;
-import org.tmatesoft.svn.core.io.SVNRepository;
-import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
-import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
-import org.tmatesoft.svn.core.wc.SVNWCUtil;
/**
* This class is responsible for the interface with ansible, retrieving log,
@@ -72,12 +47,12 @@ public class AnsibleWorker {
this.ensureWorkStructure();
***REMOVED***
- public File getWorkdir() {
- return this.workerRoot;
- ***REMOVED***
+***REMOVED*** public File getWorkdir() {
+***REMOVED*** return this.workerRoot;
+***REMOVED*** ***REMOVED***
public File getRolesDir() {
- return new File(this.getWorkdir(), ROLES_DIR);
+ return new File(this.workerRoot, ROLES_DIR);
***REMOVED***
public String getWorkerId() {
@@ -86,20 +61,20 @@ public class AnsibleWorker {
public void ensureWorkStructure() {
***REMOVED*** generate root
- this.getWorkdir().mkdirs();
+ this.workerRoot.mkdirs();
***REMOVED***
public void removeWorkStructure() {
***REMOVED*** remove the working dir
- this.getWorkdir().delete();
+ this.workerRoot.delete();
***REMOVED***
public File getPlaybookFile() {
- return new File(this.getWorkdir(), PLAYBOOK_NAME);
+ return new File(this.workerRoot, PLAYBOOK_NAME);
***REMOVED***
public File getInventoryFile() {
- return new File(this.getWorkdir(), INVENTORY_NAME);
+ return new File(this.workerRoot, INVENTORY_NAME);
***REMOVED***
@@ -122,7 +97,7 @@ public class AnsibleWorker {
public int execute(PrintStream ps)
throws IOException, InterruptedException, SVNException {
- System.out.println(this.getWorkdir());
+ System.out.println(this.workerRoot);
try {
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath()
+ " " + this.getPlaybookFile().getAbsolutePath());
@@ -149,6 +124,7 @@ public class AnsibleWorker {
while (sc.hasNextLine()) {
dest.println(sc.nextLine());
***REMOVED***
+ sc.close();
***REMOVED***
***REMOVED***).start();
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
index 403ea41..603030c 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
@@ -1,10 +1,7 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge;
import java.io.File;
-import java.io.FileOutputStream;
***REMOVED***
-import java.io.InputStream;
-import java.io.PrintStream;
import java.util.Collection;
***REMOVED***
import java.util.Map;
@@ -32,21 +29,19 @@ import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
-***REMOVED***
-import org.tmatesoft.svn.core.SVNException;
public class AnsibleBridge {
- private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class);
+ ***REMOVED***private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class);
private String dpmRoot;
- public AnsibleBridge() {
- this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager");
- ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager"));
-
- ***REMOVED***
+***REMOVED*** public AnsibleBridge() {
+***REMOVED*** this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager");
+***REMOVED*** ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager"));
+***REMOVED***
+***REMOVED*** ***REMOVED***
public AnsibleBridge(String root) {
this.dpmRoot = root;
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java
index 1302b31..20c64b6 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java
@@ -5,34 +5,71 @@ import java.io.FileOutputStream;
***REMOVED***
import java.io.PrintStream;
import java.io.PrintWriter;
-***REMOVED***
-***REMOVED***
+import java.util.Collection;
import java.util.UUID;
+***REMOVED***
+***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException;
+***REMOVED***
+***REMOVED***
import org.tmatesoft.svn.core.SVNException;
public abstract class DMPMJob {
+ private Configuration configuration;
+ private String dmpmHomeDirectory;
+ private SVNUpdater svnUpdater;
+ private File jobLogs;
+ private String id;
+ private Algorithm algorithm;
+ private Cluster cluster;
+ private String vREName;
+ private String category;
+ private String algorithm_type;
+***REMOVED***
- protected SVNUpdater svnUpdater;
- protected File jobLogs;
+ private enum STATUS
+ {
+ PROGRESS ("IN PROGRESS"),
+ COMPLETED ("COMPLETED"),
+ FAILED ("FAILED");
+
+ private String status;
+
+ STATUS (String status)
+ {
+ this.status = status;
+ ***REMOVED***
+***REMOVED***
- protected String id;
+ public DMPMJob(SVNUpdater svnUpdater,Configuration configuration,Algorithm algorithm, Cluster cluster,String vREName,
+ String category, String algorithm_type){
+ this.logger = LoggerFactory.getLogger(DMPMJob.class);
- public DMPMJob(SVNUpdater svnUpdater){
+ this.configuration = configuration;
+ this.algorithm = algorithm;
+ this.cluster = cluster;
+ this.vREName = vREName;
+ this.category = category;
+ this.algorithm_type = algorithm_type;
+
this.svnUpdater = svnUpdater;
+ this.dmpmHomeDirectory = new String (System.getProperty("user.home")+File.separator+"dataminer-pool-manager");
this.id = UUID.randomUUID().toString();
***REMOVED***TODO: dmpm work directory should be loaded from configuration file
- this.jobLogs = new File(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"+File.separator+"jobs");
-
+ this.jobLogs = new File(this.dmpmHomeDirectory+File.separator+"jobs");
this.jobLogs.mkdirs();
***REMOVED***
@@ -56,7 +93,7 @@ public abstract class DMPMJob {
Cluster dataminerCluster,
boolean includeAlgorithmDependencies,
String user){
- AnsibleBridge ansibleBridge = new AnsibleBridge();
+ AnsibleBridge ansibleBridge = new AnsibleBridge(this.dmpmHomeDirectory);
try {
return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user);
***REMOVED*** catch (IOException e) {
@@ -66,14 +103,148 @@ public abstract class DMPMJob {
***REMOVED***
- protected abstract void execute();
-
- protected int executeAnsibleWorker(AnsibleWorker worker) throws IOException, InterruptedException, SVNException{
- File path = new File(worker.getWorkdir() + File.separator + "jobs");
- path.mkdirs();
+ public void setStatusInformation(STATUS exitStatus) {
- File n = new File(this.jobLogs + File.separator + this.id);
- FileOutputStream fos = new FileOutputStream(n, true);
+
+ try
+ {
+ File statusFile = new File (this.jobLogs,this.id + "_exitStatus");
+ ***REMOVED***File m = new File ( this.jobLogs + File.separator + this.id + "_exitStatus");
+ PrintWriter writer = new PrintWriter(statusFile, "UTF-8");
+ writer.println(exitStatus.status);
+ writer.close();
+ ***REMOVED*** catch (Exception e)
+ {
+ this.logger.error ("Unable to update exit status file with status "+exitStatus.status,e);
+ ***REMOVED***
+
+
+
+***REMOVED***
+
+ private void updateLogFile (File logFile, String message)
+ {
+ try
+ {
+ PrintWriter writer = new PrintWriter(logFile,"UTF-8");
+ writer.print(message);
+ writer.close();
+ ***REMOVED*** catch (Exception e)
+ {
+ this.logger.error("Unable to log the error message: "+message,e);
+ ***REMOVED***
+
+
+***REMOVED***
+
+ protected abstract void execute ();
+
+ private boolean preInstallation (SendMail sm,NotificationHelper nh, File logFile ) throws SVNException, EMailException
+ {
+
+ this.logger.debug("Checking dependencies...");
+ Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
+ this.svnUpdater.getDependencyFile(this.algorithm.getLanguage()),
+ this.algorithm.getDependencies());
+
+ if (!undefinedDependencies.isEmpty())
+ {
+ this.logger.debug("Some dependencies are not defined");
+ String message = "Following dependencies are not defined:\n";
+ for (String n : undefinedDependencies) {
+ message += "\n" + n +"\n";
+ ***REMOVED***
+ this.setStatusInformation(STATUS.FAILED);
+ String errorMessage = nh.getFailedBody(message+"\n\n"+this.buildInfo());
+ this.updateLogFile(logFile, errorMessage);
+ sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", errorMessage);
+ return false;
+ ***REMOVED***
+ else return true;
+***REMOVED***
+
+ private void installation (SendMail sm,NotificationHelper nh,CheckMethod methodChecker,File logFile ) throws Exception
+ {
+ methodChecker.deleteFiles(this.algorithm/*, env*/);
+ int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.cluster, false, "root"),logFile);
+ System.out.println("Return code= "+ret);
+
+ if (ret != 0)
+ {
+ this.logger.debug("Ansible work failed, return code "+ret);
+ this.setStatusInformation(STATUS.FAILED);
+ String errorMessage = nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo();
+ sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",errorMessage);
+ ***REMOVED***
+
+ else if (ret == 0)
+ {
+ this.logger.debug("Operation completed");
+ this.setStatusInformation(STATUS.PROGRESS);
+ this.logger.debug("Checking the method...");
+
+ if (methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get())&&(methodChecker.algoExists(this.algorithm)))
+ {
+ this.logger.debug("Method OK and algo exists");
+ System.out.println("Interface check ok!");
+ System.out.println("Both the files exist at the correct path!");
+
+ this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.vREName,this.category, this.algorithm_type,
+ this.algorithm.getFullname());
+ this.setStatusInformation(STATUS.COMPLETED);
+ sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
+ return;
+ ***REMOVED*** else
+ {
+ this.logger.debug("Operation failed");
+ this.setStatusInformation(STATUS.FAILED);
+ sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",
+ nh.getFailedBody(
+ "\n"+
+ "Installation completed but DataMiner Interface not working correctly or files "
+ + this.algorithm.getName() + ".jar and " + this.algorithm.getName()
+ + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
+
+ ***REMOVED***
+ ***REMOVED***
+***REMOVED***
+
+ protected void execute(NotificationHelper nh, CheckMethod methodChecker)
+ {
+ SendMail sm = new SendMail();
+ try {
+
+ this.logger.debug("Pre installation operations");
+ File logFile = new File(this.jobLogs,this.id);
+ ***REMOVED***File logFile = new File(this.jobLogs + File.separator + this.id);
+ boolean preInstallationResponse = preInstallation(sm, nh, logFile);
+ this.logger.debug("Pre installation operation completed with result "+preInstallationResponse);
+ if (preInstallationResponse)
+ {
+ this.logger.debug("Installation...");
+ installation(sm, nh, methodChecker, logFile);
+ this.logger.debug("Installation completed");
+ ***REMOVED***
+
+
+
+
+
+ ***REMOVED*** catch (EMailException eme)
+ {
+ this.logger.error("Operation failed and unable to send notification email",eme);
+ ***REMOVED***
+
+ catch (Exception e) {
+ e.printStackTrace();
+ ***REMOVED***
+
+***REMOVED***
+
+
+
+ protected int executeAnsibleWorker(AnsibleWorker worker, File logFile) throws IOException, InterruptedException, SVNException{
+ FileOutputStream fos = new FileOutputStream(logFile, true);
PrintStream ps = new PrintStream(fos);
***REMOVED*** File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
@@ -81,5 +252,19 @@ public abstract class DMPMJob {
return worker.execute(ps);
***REMOVED***
+
+ public String buildInfo() {
+
+ return
+ "\n"+
+ "Algorithm details:\n"+"\n"+
+ "User: "+this.algorithm.getFullname()+"\n"+
+ "Algorithm name: "+this.algorithm.getName()+"\n"+
+ "Staging DataMiner Host: "+ this.configuration.getHost()+"\n"+
+ "Caller VRE: "+ScopeProvider.instance.get()+"\n"+
+ "Target VRE: "+this.vREName+"\n";
+***REMOVED***
+
+
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
index 10c5617..1b005c5 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
@@ -8,7 +8,10 @@ import java.util.Scanner;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
-import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderProduction;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderStaging;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging;
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNException;
@@ -17,14 +20,15 @@ import org.tmatesoft.svn.core.SVNException;
public class DataminerPoolManager {
***REMOVED***
-
- private SVNUpdater svnUpdater;
+ private SVNUpdaterStaging svnUpdaterStaging;
+ private SVNUpdaterProduction svnUpdaterProduction;
public DataminerPoolManager() {
this.logger = LoggerFactory.getLogger(this.getClass());
try {
***REMOVED***TODO: read this from configuration
- this.svnUpdater = new SVNUpdater();
+ this.svnUpdaterStaging = new SVNUpdaterStaging();
+ this.svnUpdaterProduction = new SVNUpdaterProduction();
***REMOVED*** catch (SVNException e) {
e.printStackTrace();
***REMOVED***
@@ -37,9 +41,10 @@ public class DataminerPoolManager {
this.logger.debug("Algo "+algo);
this.logger.debug("Category "+category);
this.logger.debug("Algo type "+algorithm_type);
- Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
+ ClusterBuilder stagingClusterBuilder = new ClusterBuilderStaging();
+ Cluster stagingCluster = stagingClusterBuilder.getDataminerCluster();
***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
- DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/);
+ DMPMJob job = new StagingJob(this.svnUpdaterStaging, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/);
String id = job.start();
return id;
***REMOVED***
@@ -50,8 +55,9 @@ public class DataminerPoolManager {
this.logger.debug("Algo "+algo);
this.logger.debug("Category "+category);
this.logger.debug("Algo type "+algorithm_type);
- Cluster prodCluster = ClusterBuilder.getProdDataminerCluster();
- DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/);
+ ClusterBuilder productionClusterBuilder = new ClusterBuilderProduction();
+ Cluster prodCluster = productionClusterBuilder.getDataminerCluster();
+ DMPMJob job = new ProductionPublishingJob(this.svnUpdaterProduction, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/);
String id = job.start();
return id;
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
index ec3e20e..0ec7029 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
@@ -1,164 +1,44 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service;
-import java.io.File;
import java.io.FileNotFoundException;
-import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
-import java.util.Collection;
-***REMOVED***
-***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
-import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
-import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodProduction;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperProduction;
+***REMOVED***
+***REMOVED***
public class ProductionPublishingJob extends DMPMJob {
- private Algorithm algorithm;
- private Cluster prodCluster;
- private String targetVREName;
- private String category;
- private String algorithm_type;
+
***REMOVED***private String targetVREToken;
***REMOVED***private String env;
-
+***REMOVED***
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm,
Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException {
- super(svnUpdater);
- this.jobLogs = new File(
- System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
- this.jobLogs.mkdirs();
- this.algorithm = algorithm;
- this.prodCluster = prodCluster;
- this.targetVREName = targetVREName;
- this.category = category;
- this.algorithm_type = algorithm_type;
- ***REMOVED***this.targetVREToken = targetVREToken;
- ***REMOVED***this.env= env;
- this.getStatus(0);
+ super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(),algorithm,prodCluster,targetVREName,category,algorithm_type);
+ this.logger = LoggerFactory.getLogger(StagingJob.class);***REMOVED*** this.jobLogs = new File(
+
***REMOVED***
@Override
protected void execute() {
- SendMail sm = new SendMail();
- NotificationHelper nh = new NotificationHelper();
- CheckMethod b = new CheckMethod();
- try {
-
- Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
- this.svnUpdater.getDependencyFileProd(this.algorithm.getLanguage()/*,env*/),
- this.algorithm.getDependencies());
-
- if (!undefinedDependencies.isEmpty()) {
-
- String message = "Following dependencies are not defined:\n";
- for (String n : undefinedDependencies) {
- message += "\n" + n +"\n";
- ***REMOVED***
- this.getStatus(2);
-
- sm.sendNotification(nh.getFailedSubjectRelease() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo()));
- return;
- ***REMOVED***
- b.deleteFilesProd(this.algorithm);
-
- int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.prodCluster, false, "root"));
-
- if (ret != 0) {
- this.getStatus(2);
- sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo());
- return;
- ***REMOVED***
-
-
- if (ret == 0) {
- this.getStatus(0);
-
- if (b.checkMethod(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost(), SecurityTokenProvider.instance.get())&&(b.algoExistsProd(this.algorithm/*, env*/))) {
-
- System.out.println("Interface check ok!");
- System.out.println("Both the files exist at the correct path!");
-
- this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName,this.category, this.algorithm_type,
- this.algorithm.getFullname()/*, env*/);
-
- this.getStatus(9);
- sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
- return;
- ***REMOVED*** else
- this.getStatus(2);
- sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm",
- nh.getFailedBody(
- "\n"+
- "Installation completed but DataMiner Interface not working correctly or files "
- + this.algorithm.getName() + ".jar and " + this.algorithm.getName()
- + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
- return;
-
- ***REMOVED***
- ***REMOVED*** catch (Exception e) {
- try {
- this.getStatus(0);
- ***REMOVED*** catch (FileNotFoundException e1) {
- ***REMOVED*** TODO Auto-generated catch block
- e1.printStackTrace();
- ***REMOVED*** catch (UnsupportedEncodingException e1) {
- ***REMOVED*** TODO Auto-generated catch block
- e1.printStackTrace();
- ***REMOVED***
- e.printStackTrace();
- ***REMOVED***
+ this.logger.debug("Executing staging job...");
+ super.execute(new NotificationHelperProduction(), new CheckMethodProduction());
***REMOVED***
-
- public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException {
- File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
- PrintWriter writer = new PrintWriter(m, "UTF-8");
-
- String response = "";
-
- if (exitstatus == 0) {
- response = "IN PROGRESS";
- writer.println(response);
- ***REMOVED***writer.close();
- ***REMOVED***
-
- if (exitstatus == 9) {
- response = "COMPLETED";
- writer.println(response);
- ***REMOVED***writer.close();
- ***REMOVED***
-
- if (exitstatus == 2) {
- response = "FAILED";
- writer.println(response);
- ***REMOVED***writer.close();
- ***REMOVED***
- writer.close();
- return response;
-***REMOVED***
- public String buildInfo() throws UnsupportedEncodingException{
-
- return
- "\n"+
- "Algorithm details:\n"+"\n"+
- "User: "+this.algorithm.getFullname()+"\n"+
- "Algorithm name: "+this.algorithm.getName()+"\n"+
- "Staging DataMiner Host: "+ DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost()+"\n"+
- "Caller VRE: "+ScopeProvider.instance.get()+"\n"+
- "Target VRE: "+this.targetVREName+"\n";
-***REMOVED***
+
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
index 1d8e015..5809928 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
@@ -1,184 +1,42 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service;
-import java.io.File;
import java.io.FileNotFoundException;
-import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
-import java.util.Collection;
-***REMOVED***
-***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
-import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
-import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodStaging;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperStaging;
***REMOVED***
***REMOVED***
public class StagingJob extends DMPMJob {
- private Algorithm algorithm;
- private Cluster stagingCluster;
- ***REMOVED*** private Cluster rProtoCluster;
- private String rProtoVREName;
- ***REMOVED***private String env;
- private String category;
- private String algorithm_type;
+
***REMOVED***
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm,
Cluster stagingCluster, /* Cluster rProtoCluster, */
String rProtoVREName, String category, String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException {
- super(svnUpdater);
+ super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(),algorithm,stagingCluster,rProtoVREName,category,algorithm_type);
this.logger = LoggerFactory.getLogger(StagingJob.class);
-
- this.jobLogs = new File(
- System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
- this.jobLogs.mkdirs();
-
- this.algorithm = algorithm;
- this.stagingCluster = stagingCluster;
- ***REMOVED*** this.rProtoCluster = rProtoCluster;
- this.rProtoVREName = rProtoVREName;
- ***REMOVED***this.env = env;
- this.category = category;
- this.algorithm_type = algorithm_type;
-
-
- ***REMOVED***File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
- ***REMOVED***PrintWriter writer = new PrintWriter(m, "UTF-8");
- this.getStatus(0);
- ***REMOVED***writer.close();
-
***REMOVED***
@Override
protected void execute() {
this.logger.debug("Executing staging job...");
+ super.execute(new NotificationHelperStaging(), new CheckMethodStaging());
- CheckMethod methodChecker = new CheckMethod();
- SendMail sm = new SendMail();
- NotificationHelper nh = new NotificationHelper();
-
- try {
-
- this.logger.debug("Checking dependencies...");
- Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
- this.svnUpdater.getDependencyFile(this.algorithm.getLanguage()/*,env*/),
- this.algorithm.getDependencies());
-
- if (!undefinedDependencies.isEmpty())
- {
- this.logger.debug("Some dependencies are not defined");
- String message = "Following dependencies are not defined:\n";
- for (String n : undefinedDependencies) {
- message += "\n" + n +"\n";
- ***REMOVED***
- this.getStatus(2);
-
- sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo()));
- return;
- ***REMOVED***
-
- ***REMOVED***before the installation to check if the files exist
- methodChecker.deleteFiles(this.algorithm/*, env*/);
-
- int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root"));
- System.out.println("Return code= "+ret);
-
- if (ret != 0)
- {
- this.logger.debug("Ansible work failed, return code "+ret);
- this.getStatus(2);
- sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo());
- return;
- ***REMOVED***
-
- if (ret == 0)
- {
- this.logger.debug("Operation completed");
- this.getStatus(0);
- ***REMOVED***System.out.println("1 - Checking existing in env: "+ env);
-
- ***REMOVED***System.out.println("2 - Checking existing in env: "+ this.env);
- this.logger.debug("Checking the method...");
-
-
- if (methodChecker.checkMethod(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost(), SecurityTokenProvider.instance.get())&&(methodChecker.algoExists(this.algorithm)))
- {
- this.logger.debug("Method OK and algo exists");
- System.out.println("Interface check ok!");
- System.out.println("Both the files exist at the correct path!");
-
- this.svnUpdater.updateSVNStagingAlgorithmList(this.algorithm, this.rProtoVREName,this.category, this.algorithm_type,
- this.algorithm.getFullname()/*, env*/);
-
- this.getStatus(9);
- sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
- return;
- ***REMOVED*** else
- {
- this.logger.debug("Operation failed");
- this.getStatus(2);
- sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",
- nh.getFailedBody(
- "\n"+
- "Installation completed but DataMiner Interface not working correctly or files "
- + this.algorithm.getName() + ".jar and " + this.algorithm.getName()
- + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
- return;
- ***REMOVED***
- ***REMOVED***
- ***REMOVED*** catch (Exception e) {
- e.printStackTrace();
- ***REMOVED***
***REMOVED***
- public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException {
- File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
- PrintWriter writer = new PrintWriter(m, "UTF-8");
-
- String response = "";
-
- if (exitstatus == 0) {
- response = "IN PROGRESS";
- writer.println(response);
- ***REMOVED***writer.close();
- ***REMOVED***
-
- if (exitstatus == 9) {
- response = "COMPLETED";
- writer.println(response);
- ***REMOVED***writer.close();
- ***REMOVED***
-
- if (exitstatus == 2) {
- response = "FAILED";
- writer.println(response);
- ***REMOVED***writer.close();
- ***REMOVED***
- writer.close();
- return response;
-***REMOVED***
- public String buildInfo() throws UnsupportedEncodingException{
- return
- "\n"+
- "Algorithm details:\n"+"\n"+
- "User: "+this.algorithm.getFullname()+"\n"+
- "Algorithm name: "+this.algorithm.getName()+"\n"+
- "Staging DataMiner Host: "+ DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost()+"\n"+
- "Caller VRE: "+ScopeProvider.instance.get()+"\n"+
- "Target VRE: "+rProtoVREName+"\n";
-***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java
index 4680325..173a971 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java
@@ -10,20 +10,11 @@ import java.util.Arrays;
***REMOVED***
***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
-import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
***REMOVED***
-import com.jcraft.jsch.Channel;
-import com.jcraft.jsch.ChannelSftp;
-import com.jcraft.jsch.JSch;
-import com.jcraft.jsch.Session;
-import com.jcraft.jsch.SftpATTRS;
-import com.jcraft.jsch.SftpException;
-
-public class CheckMethod {
+public abstract class CheckMethod {
***REMOVED***
@@ -37,14 +28,14 @@ public class CheckMethod {
this.logger.debug("Checking method for machine "+machine);
this.logger.debug("By using tocken "+token);
System.out.println("Machine: " + machine);
- String getCapabilitesRequest = new String();
- String getCapabilitesResponse = new String();
+***REMOVED*** String getCapabilitesRequest = new String();
+***REMOVED*** String getCapabilitesResponse = new String();
System.out.println(" Token: " + token);
String request = "http:***REMOVED***" + machine
+ "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token;
String response = machine + "___" + token + ".xml";
- getCapabilitesRequest = request;
- getCapabilitesResponse = response;
+***REMOVED*** getCapabilitesRequest = request;
+***REMOVED*** getCapabilitesResponse = response;
String baseDescriptionRequest = "http:***REMOVED***" + machine
+ "/wps/WebProcessingService?Request=DescribeProcess&Service=WPS&Version=1.0.0" + "&gcube-token="
+ token + "&Identifier=";
@@ -111,359 +102,22 @@ public class CheckMethod {
- public boolean algoExists(Algorithm algo/*, String env*/) throws Exception{
+ public abstract boolean algoExists(Algorithm algo/*, String env*/) throws Exception;
- this.logger.debug("Looking if algo "+algo.getName()+ " exists");
- Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
- File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
- File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
- this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath());
-
- System.out.println("First file is located to: "+file.getPath());
- System.out.println("Second file is located to: "+file2.getPath());
-
-
- if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))
- {
- this.logger.debug("Files found");
- this.copyFromDmToSVN(file/*,env*/);
- this.copyFromDmToSVN(file2/*,env*/);
- System.out.println("Files have been copied to SVN");
-
-
- return true;
- ***REMOVED***
- else
- {
- this.logger.debug("Files not found");
- System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
- return false;
- ***REMOVED***
-
-***REMOVED***
+ public abstract void deleteFiles(Algorithm a/*,String env*/) throws Exception;
+
- public boolean algoExistsProd(Algorithm a/*, String env*/) throws Exception{
- Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
- File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
- File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
-
-
- System.out.println("First file is located to: "+file.getPath());
- System.out.println("Second file is located to: "+file2.getPath());
-
-
- if ((this.doesExistProd(file.getPath()/*,env*/)) && (this.doesExistProd(file2.getPath()/*,env*/))){
- this.copyFromDmToSVNProd(file/*,env*/);
- this.copyFromDmToSVNProd(file2/*,env*/);
-
-
- return true;
- ***REMOVED***
- else
- System.out.println("Algorithm "+a.getName()+".jar"+ " and "+a.getName()+"_interface.jar files are not present at the expected path");
- return false;
-***REMOVED***
+ public abstract boolean doesExist(String path/*, String env*/) throws Exception;
+
- public void deleteFiles(Algorithm a/*,String env*/) throws Exception{
- JSch jsch = new JSch();
- Session session = null;
- Channel channel = null;
- ChannelSftp c = null;
- Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
- System.out.println("checking existing in env: " + stagingConfiguration.getHost());
-
- File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
- File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
-
-
- System.out.println("First file is located to: "+file.getPath());
- System.out.println("Second file is located to: "+file2.getPath());
-
-
- jsch.setKnownHosts("~/.ssh/known_hosts");
- String privateKey = "~/.ssh/id_rsa";
-
- jsch.addIdentity(privateKey);
- System.out.println("Private Key Added.");
-
- session = jsch.getSession("root", stagingConfiguration.getHost());
- System.out.println("session created.");
-
- java.util.Properties config = new java.util.Properties();
- config.put("StrictHostKeyChecking", "no");
-
- session.setConfig(config);
- session.connect();
-
- channel = session.openChannel("sftp");
- channel.connect();
- System.out.println("shell channel connected....");
-
- c = (ChannelSftp) channel;
-
- if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){
-
- c.rm(file.getPath());
- c.rm(file2.getPath());
- System.out.println("Both the files have been deleted");
- ***REMOVED***
- else System.out.println("Files not found");
- channel.disconnect();
- c.disconnect();
- session.disconnect();
-
-***REMOVED***
-
-
- public void deleteFilesProd(Algorithm a/*,String env*/) throws Exception{
- JSch jsch = new JSch();
- Session session = null;
- Channel channel = null;
- ChannelSftp c = null;
- Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
- System.out.println("checking existing in env: " + productionConfiguration.getHost());
-
- File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
- File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
-
-
- System.out.println("First file is located to: "+file.getPath());
- System.out.println("Second file is located to: "+file2.getPath());
-
-
- jsch.setKnownHosts("~/.ssh/known_hosts");
- String privateKey = "~/.ssh/id_rsa";
-
- jsch.addIdentity(privateKey);
- System.out.println("Private Key Added.");
-
- session = jsch.getSession("root", productionConfiguration.getHost());
- System.out.println("session created.");
-
- java.util.Properties config = new java.util.Properties();
- config.put("StrictHostKeyChecking", "no");
-
- session.setConfig(config);
- session.connect();
-
- channel = session.openChannel("sftp");
- channel.connect();
- System.out.println("shell channel connected....");
-
- c = (ChannelSftp) channel;
-
- if(doesExistProd(file.getPath()/*,env*/)&&(doesExistProd(file2.getPath()/*,env*/))){
-
- c.rm(file.getPath());
- c.rm(file2.getPath());
- System.out.println("Both the files have been deleted");
- ***REMOVED***
- else System.out.println("Files not found");
- channel.disconnect();
- c.disconnect();
- session.disconnect();
-
-***REMOVED***
-
-
-
- public boolean doesExist(String path/*, String env*/) throws Exception {
- this.logger.debug("Looking if file "+path + " exists");
- JSch jsch = new JSch();
- Session session = null;
- Channel channel = null;
- ChannelSftp c = null;
- boolean success = false;
- Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
- jsch.setKnownHosts("~/.ssh/known_hosts");
- String privateKey = "~/.ssh/id_rsa";
-
- jsch.addIdentity(privateKey);
- System.out.println("Private Key Added.");
-
- this.logger.debug("Staging configuration host "+stagingConfiguration.getHost());
- session = jsch.getSession("root",stagingConfiguration.getHost() );
- System.out.println("session created.");
-
- java.util.Properties config = new java.util.Properties();
- config.put("StrictHostKeyChecking", "no");
-
- session.setConfig(config);
- session.connect();
-
- channel = session.openChannel("sftp");
- channel.connect();
- System.out.println("shell channel connected....");
-
- c = (ChannelSftp) channel;
-
- SftpATTRS is = null;
- System.out.println(path);
-
- try {
- is = c.lstat(path);
- this.logger.debug("File found");
- success = true;
- ***REMOVED*** catch (SftpException e)
- {
- this.logger.error("File not found",e);
-
- if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE)
- {
- ***REMOVED*** file doesn't exist
- success = false;
- ***REMOVED***
- ***REMOVED***success = true; ***REMOVED*** something else went wrong
- ***REMOVED***
- channel.disconnect();
- c.disconnect();
- session.disconnect();
- this.logger.debug("Operation result "+success);
- return success;
-
-***REMOVED***
-
-
-
- public boolean doesExistProd(String path/*, String env*/) throws Exception {
- JSch jsch = new JSch();
- Session session = null;
- Channel channel = null;
- ChannelSftp c = null;
- boolean success = false;
- Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
-
-
- jsch.setKnownHosts("~/.ssh/known_hosts");
- String privateKey = "~/.ssh/id_rsa";
-
- jsch.addIdentity(privateKey);
- System.out.println("Private Key Added.");
-
- session = jsch.getSession("root", productionConfiguration.getHost());
- System.out.println("session created.");
-
- java.util.Properties config = new java.util.Properties();
- config.put("StrictHostKeyChecking", "no");
-
- session.setConfig(config);
- session.connect();
-
- channel = session.openChannel("sftp");
- channel.connect();
- System.out.println("shell channel connected....");
-
- c = (ChannelSftp) channel;
-
- SftpATTRS is = null;
- System.out.println(path);
-
- try {
- is = c.lstat(path);
- success = true;
- ***REMOVED*** catch (SftpException e) {
- if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
- ***REMOVED*** file doesn't exist
- success = false;
- ***REMOVED***
- ***REMOVED***success = true; ***REMOVED*** something else went wrong
- ***REMOVED***
- channel.disconnect();
- c.disconnect();
- session.disconnect();
- return success;
-
-***REMOVED***
-
-
- public void copyFromDmToSVN(File a/*,String env*/) throws Exception {
- JSch jsch = new JSch();
- Session session = null;
- SVNUpdater svnUpdater = new SVNUpdater();
- Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
-
-
- jsch.setKnownHosts("~/.ssh/known_hosts");
- String privateKey = "~/.ssh/id_rsa";
-
- jsch.addIdentity(privateKey);
- System.out.println("Private Key Added.");
-
- session = jsch.getSession("root", stagingConfiguration.getHost());
- System.out.println("session created.");
-
- java.util.Properties config = new java.util.Properties();
- config.put("StrictHostKeyChecking", "no");
-
- session.setConfig(config);
- session.connect();
-
- Channel channel = session.openChannel("sftp");
- channel.connect();
-
- ChannelSftp sftp = (ChannelSftp) channel;
-
- sftp.cd(stagingConfiguration.getGhostAlgoDirectory());
-
- System.out.println("REMOTE : "+stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
- System.out.println("LOCAL : /tmp/"+a.getName());
-
- sftp.get(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
-
- channel.disconnect();
- session.disconnect();
-
- File f = new File("/tmp/"+a.getName());
- svnUpdater.updateAlgorithmFiles(f);
- f.delete();
-***REMOVED***
+ public abstract void copyFromDmToSVN(File a/*,String env*/) throws Exception;
-
- public void copyFromDmToSVNProd(File a/*,String env*/) throws Exception {
- JSch jsch = new JSch();
- Session session = null;
- SVNUpdater svnUpdater = new SVNUpdater();
- Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
- jsch.setKnownHosts("~/.ssh/known_hosts");
- String privateKey = "~/.ssh/id_rsa";
-
- jsch.addIdentity(privateKey);
- System.out.println("Private Key Added.");
-
- session = jsch.getSession("root", productionConfiguration.getHost());
- System.out.println("session created.");
-
- java.util.Properties config = new java.util.Properties();
- config.put("StrictHostKeyChecking", "no");
-
- session.setConfig(config);
- session.connect();
-
- Channel channel = session.openChannel("sftp");
- channel.connect();
-
- ChannelSftp sftp = (ChannelSftp) channel;
-
- sftp.cd(productionConfiguration.getGhostAlgoDirectory());
-
- System.out.println("REMOTE : "+productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
- System.out.println("LOCAL : /tmp/"+a.getName());
-
- sftp.get(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
-
- channel.disconnect();
- session.disconnect();
-
- File f = new File("/tmp/"+a.getName());
- svnUpdater.updateAlgorithmFilesProd(f);
- f.delete();
-***REMOVED***
- public List getFiles(String a){
+ public static List getFiles(String a){
String[] array = a.split(",");
ArrayList list = new ArrayList<>(Arrays.asList(array));
@@ -477,71 +131,5 @@ public class CheckMethod {
***REMOVED***
- public static void main(String[] args) throws Exception {
-***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
-***REMOVED*** System.out.println(a.getStagingHost());
-
- CheckMethod a = new CheckMethod();
-
- ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
-
-***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar");
-***REMOVED*** System.out.println(aa.getName());
-***REMOVED*** System.out.println(aa.getPath());
-
-
-
-
-
-
- ***REMOVED***a.copyFromDmToSVN(aa);
-***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")){
-***REMOVED*** System.out.println("AAA");***REMOVED***
-***REMOVED***
-***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){
-***REMOVED*** System.out.println("BBBB");
-***REMOVED***
-***REMOVED******REMOVED***
-***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){
-***REMOVED*** System.out.println("CCCC");***REMOVED***
-***REMOVED***
-***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar");
-***REMOVED*** a.copyFromDmToSVN(aa, "Dev");
-
-***REMOVED***
-
-
-***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org",
-***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548"));
-
-***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
- ***REMOVED*** "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"));
-
-System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org",
- "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"));
-
-
-***REMOVED***Algorithm aa = new Algorithm();
-***REMOVED***aa.setName("UDPIPE_WRAPPER");
-***REMOVED***System.out.println(a.algoExists(aa));
-***REMOVED******REMOVED***
-***REMOVED***ServiceConfiguration bp = new ServiceConfiguration();
-***REMOVED******REMOVED***
-***REMOVED***SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
-***REMOVED******REMOVED***
-***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); {
-***REMOVED***System.out.println("ciao");
-***REMOVED***
-***REMOVED******REMOVED***
***REMOVED***
-***REMOVED***Algorithm al = new Algorithm();
-***REMOVED*** al.setName("UDPIPE_WRAPPER");
-***REMOVED*** a.deleteFiles(al);
-
-
-
-
-
-***REMOVED***
-***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java
index fe70170..23fddb9 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java
@@ -1,11 +1,9 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
+import static org.gcube.common.authorization.client.Constants.authorizationService;
+
***REMOVED***
import org.gcube.common.authorization.library.AuthorizationEntry;
-***REMOVED***
-***REMOVED***
-
-import static org.gcube.common.authorization.client.Constants.authorizationService;
public class CheckPermission {
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
index f904e47..b6d02a0 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
@@ -6,58 +6,37 @@ import java.io.FileNotFoundException;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
-import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
***REMOVED***
***REMOVED***
-public class ClusterBuilder {
+public abstract class ClusterBuilder {
+ private Configuration configuration;
+
+ public ClusterBuilder (Configuration configuration)
+ {
+ this.configuration = configuration;
+***REMOVED***
***REMOVED***1. to complete
- public static Cluster getStagingDataminerCluster() throws FileNotFoundException{
+ public Cluster getDataminerCluster() throws FileNotFoundException{
Cluster cluster = new Cluster();
Host h = new Host();
- h.setName(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost());
+ h.setName(this.configuration.getHost());
cluster.addHost(h);
-***REMOVED*** if (env.equals("Dev")){
-***REMOVED*** h.setName(p.getDevStagingHost());
-***REMOVED*** cluster.addHost(h);
-***REMOVED*** ***REMOVED***
-***REMOVED***
-***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto")))){
-***REMOVED*** h.setName(p.getProtoProdStagingHost());
-***REMOVED*** cluster.addHost(h);
-***REMOVED*** ***REMOVED***
-
- return cluster;
-***REMOVED***
-
-
- public static Cluster getProdDataminerCluster() throws FileNotFoundException{
- Cluster cluster = new Cluster();
- Host h = new Host();
- h.setName(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost());
- cluster.addHost(h);
-
-***REMOVED*** if (env.equals("Dev")){
-***REMOVED*** h.setName(p.getDevStagingHost());
-***REMOVED*** cluster.addHost(h);
-***REMOVED*** ***REMOVED***
-***REMOVED***
-***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto")))){
-***REMOVED*** h.setName(p.getProtoProdStagingHost());
-***REMOVED*** cluster.addHost(h);
-***REMOVED*** ***REMOVED***
return cluster;
***REMOVED***
+
- public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException{
+
+ public Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException{
Cluster cluster = new Cluster();
for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) {
cluster.addHost(h);
@@ -65,13 +44,13 @@ public class ClusterBuilder {
return cluster;
***REMOVED***
- public static Cluster getRProtoCluster() throws IOException{
+ public Cluster getRProtoCluster() throws IOException{
***REMOVED***Assumes the service is running in RPrototypingLab
String token = SecurityTokenProvider.instance.get();
String targetVRE = ScopeProvider.instance.get();
- return getVRECluster(token, targetVRE);
+ return this.getVRECluster(token, targetVRE);
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java
index d64e86d..ff686c4 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java
@@ -1,53 +1,28 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
-import java.text.SimpleDateFormat;
-import java.util.Calendar;
-
-import org.apache.commons.lang.StringUtils;
-
***REMOVED***import scala.actors.threadpool.Arrays;
-public class NotificationHelper {
+public abstract class NotificationHelper {
- private Exception executionException;
+***REMOVED*** private Exception executionException;
- public NotificationHelper() {
- ***REMOVED***
- private String getSubjectHeader() {
- return "[DataMinerGhostStagingInstallationRequestReport]";
- ***REMOVED***
+
+
+***REMOVED*** private boolean isError() {
+***REMOVED*** return this.executionException!=null;
+***REMOVED*** ***REMOVED***
- private String getSubjectHeaderRelease() {
- return "[DataMinerGhostProductionInstallationRequestReport]";
- ***REMOVED***
+***REMOVED*** public void setExecutionException(Exception executionException) {
+***REMOVED*** this.executionException = executionException;
+***REMOVED*** ***REMOVED***
- private boolean isError() {
- return this.executionException!=null;
- ***REMOVED***
-
- public void setExecutionException(Exception executionException) {
- this.executionException = executionException;
- ***REMOVED***
+ public abstract String getSuccessSubject();
- public String getSuccessSubject() {
- return this.getSubjectHeader()+" is SUCCESS";
- ***REMOVED***
- public String getSuccessSubjectRelease() {
- return this.getSubjectHeaderRelease()+" is SUCCESS";
- ***REMOVED***
-
-
- public String getFailedSubjectRelease() {
- return this.getSubjectHeaderRelease()+" is FAILED";
- ***REMOVED***
-
+ public abstract String getFailedSubject();
- public String getFailedSubject() {
- return String.format(this.getSubjectHeader()+" is FAILED");
- ***REMOVED***
-
+
public String getSuccessBody(String info) {
String message = String.format("The installation of the algorithm is completed successfully.");
message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info;
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java
new file mode 100755
index 0000000..d684a12
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java
@@ -0,0 +1,38 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
+import org.tmatesoft.svn.core.SVNException;
+import org.tmatesoft.svn.core.SVNURL;
+import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
+import org.tmatesoft.svn.core.io.SVNRepository;
+import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
+import org.tmatesoft.svn.core.wc.SVNWCUtil;
+
+public class SVNRepositoryManager {
+
+ private SVNRepository svnRepository;
+ private static SVNRepositoryManager instance;
+
+ private SVNRepositoryManager (Configuration configuration) throws SVNException
+ {
+ this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
+ ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
+ this.svnRepository.setAuthenticationManager(authManager);
+***REMOVED***
+
+
+ public static SVNRepositoryManager getInstance (Configuration configuration) throws SVNException
+ {
+ if (instance == null) instance = new SVNRepositoryManager(configuration);
+
+ return instance;
+***REMOVED***
+
+
+ public SVNRepository getSvnRepository() {
+ return svnRepository;
+***REMOVED***
+
+
+
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
index aa813eb..b937af5 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
@@ -21,33 +21,28 @@ import java.util.TimeZone;
import org.apache.commons.io.FileUtils;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
-import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNNodeKind;
-import org.tmatesoft.svn.core.SVNURL;
-import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
import org.tmatesoft.svn.core.io.ISVNEditor;
import org.tmatesoft.svn.core.io.SVNRepository;
-import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
-import org.tmatesoft.svn.core.wc.SVNWCUtil;
/**
* Created by ggiammat on 5/9/17.
*/
-public class SVNUpdater {
+public abstract class SVNUpdater {
private SVNRepository svnRepository;
+ private Configuration configuration;
-
- public SVNUpdater() throws SVNException {
- this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getSVNRepository()));
- ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
- this.svnRepository.setAuthenticationManager(authManager);
+ public SVNUpdater(Configuration configuration) throws SVNException {
+ this.configuration = configuration;
+ this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository();
+
***REMOVED***
***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) {
@@ -58,113 +53,14 @@ public class SVNUpdater {
- public String getDependencyFile(String language/*, String env*/){
-
- String a = "";
-***REMOVED*** if (env.equals("Dev")){
-***REMOVED*** a= this.getDevDependencyFile(language);
-***REMOVED*** ***REMOVED***
-***REMOVED***
-***REMOVED*** if (env.equals("Prod")){
-***REMOVED*** a= this.getProdDependencyFile(language);
-***REMOVED*** ***REMOVED***
-***REMOVED***
-***REMOVED*** if (env.equals("Proto")){
-***REMOVED*** a= this.getRProtoDependencyFile(language);
-***REMOVED*** ***REMOVED***
-***REMOVED***
-***REMOVED*** if (env.equals("Preprod")){
-***REMOVED*** a= this.getPreProdDependencyFile(language);
-***REMOVED*** ***REMOVED***
- a = this.getStagingDependencyFile(language);
- return a;
+ public String getDependencyFile(String language/*, String env*/)
+ {
+ return getDependencyFile(this.configuration,language);
***REMOVED***
-public String getDependencyFileProd(String language/*, String env*/){
-
- String a = "";
-***REMOVED*** if (env.equals("Dev")){
-***REMOVED*** a= this.getDevDependencyFile(language);
-***REMOVED*** ***REMOVED***
-***REMOVED***
-***REMOVED*** if (env.equals("Prod")){
-***REMOVED*** a= this.getProdDependencyFile(language);
-***REMOVED*** ***REMOVED***
-***REMOVED***
-***REMOVED*** if (env.equals("Proto")){
-***REMOVED*** a= this.getRProtoDependencyFile(language);
-***REMOVED*** ***REMOVED***
-***REMOVED***
-***REMOVED*** if (env.equals("Preprod")){
-***REMOVED*** a= this.getPreProdDependencyFile(language);
-***REMOVED*** ***REMOVED***
- a = this.getProdDependencyFile(language);
- return a;
-***REMOVED***
-
-
-
- public String getStagingDependencyFile(String language) {
-
- return getDependencyFile(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(),language);
-
-***REMOVED*** switch (language) {
-***REMOVED*** case "R":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingCRANDepsList();
-***REMOVED*** case "R-blackbox":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingRBDepsList();
-***REMOVED*** case "Java":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingJavaDepsList();
-***REMOVED*** case "Knime-Workflow":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingKWDepsList();
-***REMOVED*** case "Linux-compiled":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingLinuxCompiledDepsList();
-***REMOVED*** case "Octave":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingOctaveDepsList();
-***REMOVED*** case "Python":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingPythonDepsList();
-***REMOVED*** case "Pre-Installed":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingPreInstalledDepsList();
-***REMOVED*** case "Windows-compiled":
-***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingWCDepsList();
-***REMOVED*** default:
-***REMOVED*** return null;
-***REMOVED*** ***REMOVED***
-***REMOVED***
-
-
- public String getProdDependencyFile(String language) {
-
-
- return getDependencyFile(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(),language);
-
-***REMOVED*** switch (language) {
-***REMOVED*** case "R":
-***REMOVED*** return this.configuration.getSVNProdCRANDepsList();
-***REMOVED*** case "R-blackbox":
-***REMOVED*** return this.configuration.getSVNProdRBDepsList();
-***REMOVED*** case "Java":
-***REMOVED*** return this.configuration.getSVNProdJavaDepsList();
-***REMOVED*** case "Knime-Workflow":
-***REMOVED*** return this.configuration.getSVNProdKWDepsList();
-***REMOVED*** case "Linux-compiled":
-***REMOVED*** return this.configuration.getSVNProdLinuxCompiledDepsList();
-***REMOVED*** case "Octave":
-***REMOVED*** return this.configuration.getSVNProdOctaveDepsList();
-***REMOVED*** case "Python":
-***REMOVED*** return this.configuration.getSVNProdPythonDepsList();
-***REMOVED*** case "Pre-Installed":
-***REMOVED*** return this.configuration.getSVNProdPreInstalledDepsList();
-***REMOVED*** case "Windows-compiled":
-***REMOVED*** return this.configuration.getSVNProdWCDepsList();
-***REMOVED*** default:
-***REMOVED*** return null;
-***REMOVED*** ***REMOVED***
-***REMOVED***
-
private String getDependencyFile (Configuration configuration, String language)
{
@@ -193,29 +89,24 @@ public String getDependencyFileProd(String language/*, String env*/){
***REMOVED***
- public void updateSVNStagingAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
- this.updateSVNAlgorithmList(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
-***REMOVED***
-
- public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
- this.updateSVNAlgorithmList(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
+ public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/)
+ {
+ this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
***REMOVED***
+
public void updateAlgorithmFiles(File a) throws SVNException{
***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a);
- this.updateAlgorithmList(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getRepository(), a);
+ this.updateAlgorithmList(this.configuration.getRepository(), a);
***REMOVED***
- public void updateAlgorithmFilesProd(File a) throws SVNException{
- ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a);
- this.updateAlgorithmList(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getRepository(), a);
-***REMOVED***
+
- public void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException {
+ private void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException {
try {
System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo);
@@ -312,7 +203,7 @@ public String getDependencyFileProd(String language/*, String env*/){
***REMOVED***
- public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
+ private void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
try {
System.out.println("Updating algorithm list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@@ -404,10 +295,10 @@ public String getDependencyFileProd(String language/*, String env*/){
System.out.println("Checking dependencies list: " + file);
- CheckMethod cm = new CheckMethod();
+
List validDependencies = new LinkedList();
- for (String singlefile: cm.getFiles(file)){
+ for (String singlefile: CheckMethod.getFiles(file)){
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(singlefile, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
@@ -477,8 +368,8 @@ public String getDependencyFileProd(String language/*, String env*/){
***REMOVED***
public boolean checkIfAvaialable(String file, Collection deps) throws SVNException {
- SendMail sm = new SendMail();
- NotificationHelper nh = new NotificationHelper();
+ ***REMOVED***SendMail sm = new SendMail();
+ ***REMOVED***NotificationHelper nh = new NotificationHelper();
boolean check = false;
try {
@@ -608,7 +499,7 @@ public String getDependencyFileProd(String language/*, String env*/){
- SVNUpdater c = new SVNUpdater();
+ ***REMOVED*** SVNUpdater c = new SVNUpdater();
***REMOVED***File a = new File("/home/ngalante/Desktop/testCiro");
***REMOVED***File b = new File ("/home/ngalante/Desktop/testB");
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java
index b5dc261..c5ac255 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java
@@ -8,6 +8,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
***REMOVED***
***REMOVED***
import java.io.OutputStreamWriter;
+import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
***REMOVED***
import java.net.URLConnection;
@@ -23,16 +24,10 @@ import org.apache.http.util.EntityUtils;
import org.gcube.common.authorization.library.AuthorizationEntry;
***REMOVED***
import org.gcube.common.resources.gcore.GCoreEndpoint;
-import org.gcube.common.resources.gcore.GenericResource;
-import org.gcube.common.resources.gcore.ServiceEndpoint;
***REMOVED***
-
-***REMOVED***
-
+import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
-import org.gcube.smartgears.ContextProvider;
-import org.gcube.smartgears.context.application.ApplicationContext;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -43,12 +38,14 @@ public class SendMail {
***REMOVED*** TODO Auto-generated constructor stub
***REMOVED***
- public void sendNotification(String subject, String body) throws Exception {
+ public void sendNotification(String subject, String body) throws EMailException {
***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get());
***REMOVED***String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get());
+
+
String serviceAddress = this.getSocialService();
@@ -62,18 +59,25 @@ public class SendMail {
***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage);
***REMOVED*** put the sender, the recipients, subject and body of the mail here
- subject = URLEncoder.encode(subject, "UTF-8");
- body = URLEncoder.encode(body, "UTF-8");
+
+ try
+ {
+ subject = URLEncoder.encode(subject, "UTF-8");
+ body = URLEncoder.encode(body, "UTF-8");
+ ***REMOVED***
+ catch (UnsupportedEncodingException e)
+ {
+ throw new EMailException(e);
+ ***REMOVED***
+
String requestParameters = "&sender=dataminer&recipients=" + this.getAdmins() + "&subject=" + subject + "&body="
+ body;
String response = this.sendPostRequest(requestForMessage, requestParameters);
***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK ");
- if (response == null) {
- Exception e = new Exception("Error in email sending response");
- throw e;
- ***REMOVED***
+ if (response == null) throw new EMailException();
+
***REMOVED***
***REMOVED*** public void notifySubmitter(String a, String b) throws Exception {
@@ -191,7 +195,7 @@ public class SendMail {
- public String getAdmins() throws Exception {
+ public String getAdmins(){
try{
List s = new LinkedList();
JSONObject obj = new JSONObject(this.getAdminRoles());
@@ -206,7 +210,7 @@ public class SendMail {
***REMOVED***
return s.toString().replace("[", "").replace("]", "");
***REMOVED***
- catch(Exception a){return "ngalante, lucio.lelii, roberto.cirillo, gianpaolo.coro, giancarlo.panichi, scarponi"; ***REMOVED***
+ catch(Exception a){return "ciro.formisano, lucio.lelii, roberto.cirillo, gianpaolo.coro, giancarlo.panichi, scarponi"; ***REMOVED***
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java
new file mode 100755
index 0000000..2392db6
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java
@@ -0,0 +1,19 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
+
+public class EMailException extends Exception {
+
+
+
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ public EMailException() {
+ super ("Unable to send email notification");
+***REMOVED***
+
+ public EMailException(Throwable cause) {
+ super ("Unable to send email notification",cause);
+***REMOVED***
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java
new file mode 100755
index 0000000..9967ccc
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java
@@ -0,0 +1,272 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
+
+import java.io.File;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+***REMOVED***
+***REMOVED***
+
+import com.jcraft.jsch.Channel;
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.Session;
+import com.jcraft.jsch.SftpException;
+
+public class CheckMethodProduction extends CheckMethod{
+
+***REMOVED***
+
+ public CheckMethodProduction()
+ {
+ this.logger = LoggerFactory.getLogger(CheckMethodProduction.class);
+***REMOVED***
+
+
+
+
+ @Override
+ public boolean algoExists(Algorithm algo/*, String env*/) throws Exception{
+
+ this.logger.debug("Looking if algo "+algo.getName()+ " exists in production");
+ Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
+ File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
+ File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
+
+
+ System.out.println("First file is located to: "+file.getPath());
+ System.out.println("Second file is located to: "+file2.getPath());
+
+
+ if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/))){
+ this.copyFromDmToSVN(file/*,env*/);
+ this.copyFromDmToSVN(file2/*,env*/);
+
+
+ return true;
+ ***REMOVED***
+ else
+ System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
+ return false;
+***REMOVED***
+
+ @Override
+ public void deleteFiles(Algorithm a/*,String env*/) throws Exception
+ {
+ JSch jsch = new JSch();
+ Session session = null;
+ Channel channel = null;
+ ChannelSftp c = null;
+ Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
+ System.out.println("checking existing in env: " + productionConfiguration.getHost());
+
+ File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
+ File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
+
+
+ System.out.println("First file is located to: "+file.getPath());
+ System.out.println("Second file is located to: "+file2.getPath());
+
+
+ jsch.setKnownHosts("~/.ssh/known_hosts");
+ String privateKey = "~/.ssh/id_rsa";
+
+ jsch.addIdentity(privateKey);
+ System.out.println("Private Key Added.");
+
+ session = jsch.getSession("root", productionConfiguration.getHost());
+ System.out.println("session created.");
+
+ java.util.Properties config = new java.util.Properties();
+ config.put("StrictHostKeyChecking", "no");
+
+ session.setConfig(config);
+ session.connect();
+
+ channel = session.openChannel("sftp");
+ channel.connect();
+ System.out.println("shell channel connected....");
+
+ c = (ChannelSftp) channel;
+
+ if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){
+
+ c.rm(file.getPath());
+ c.rm(file2.getPath());
+ System.out.println("Both the files have been deleted");
+ ***REMOVED***
+ else System.out.println("Files not found");
+ channel.disconnect();
+ c.disconnect();
+ session.disconnect();
+
+***REMOVED***
+
+
+ @Override
+ public boolean doesExist(String path/*, String env*/) throws Exception {
+ JSch jsch = new JSch();
+ Session session = null;
+ Channel channel = null;
+ ChannelSftp c = null;
+ boolean success = false;
+ Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
+
+
+ jsch.setKnownHosts("~/.ssh/known_hosts");
+ String privateKey = "~/.ssh/id_rsa";
+
+ jsch.addIdentity(privateKey);
+ System.out.println("Private Key Added.");
+
+ session = jsch.getSession("root", productionConfiguration.getHost());
+ System.out.println("session created.");
+
+ java.util.Properties config = new java.util.Properties();
+ config.put("StrictHostKeyChecking", "no");
+
+ session.setConfig(config);
+ session.connect();
+
+ channel = session.openChannel("sftp");
+ channel.connect();
+ System.out.println("shell channel connected....");
+
+ c = (ChannelSftp) channel;
+
+
+ System.out.println(path);
+
+ try {
+ c.lstat(path);
+ success = true;
+ ***REMOVED*** catch (SftpException e) {
+ if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
+ ***REMOVED*** file doesn't exist
+ success = false;
+ ***REMOVED***
+ ***REMOVED***success = true; ***REMOVED*** something else went wrong
+ ***REMOVED***
+ channel.disconnect();
+ c.disconnect();
+ session.disconnect();
+ return success;
+
+***REMOVED***
+
+
+ @Override
+ public void copyFromDmToSVN(File a/*,String env*/) throws Exception {
+ JSch jsch = new JSch();
+ Session session = null;
+ SVNUpdater svnUpdater = new SVNUpdaterProduction();
+ Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
+
+ jsch.setKnownHosts("~/.ssh/known_hosts");
+ String privateKey = "~/.ssh/id_rsa";
+
+ jsch.addIdentity(privateKey);
+ System.out.println("Private Key Added.");
+
+ session = jsch.getSession("root", productionConfiguration.getHost());
+ System.out.println("session created.");
+
+ java.util.Properties config = new java.util.Properties();
+ config.put("StrictHostKeyChecking", "no");
+
+ session.setConfig(config);
+ session.connect();
+
+ Channel channel = session.openChannel("sftp");
+ channel.connect();
+
+ ChannelSftp sftp = (ChannelSftp) channel;
+
+ sftp.cd(productionConfiguration.getGhostAlgoDirectory());
+
+ System.out.println("REMOTE : "+productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
+ System.out.println("LOCAL : /tmp/"+a.getName());
+
+ sftp.get(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
+
+ channel.disconnect();
+ session.disconnect();
+
+ File f = new File("/tmp/"+a.getName());
+ svnUpdater.updateAlgorithmFiles(f);
+ f.delete();
+***REMOVED***
+
+
+
+ public static void main(String[] args) throws Exception {
+***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
+***REMOVED*** System.out.println(a.getStagingHost());
+
+ CheckMethodProduction a = new CheckMethodProduction();
+
+ ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
+
+***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar");
+***REMOVED*** System.out.println(aa.getName());
+***REMOVED*** System.out.println(aa.getPath());
+
+
+
+
+
+
+ ***REMOVED***a.copyFromDmToSVN(aa);
+***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")){
+***REMOVED*** System.out.println("AAA");***REMOVED***
+***REMOVED***
+***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){
+***REMOVED*** System.out.println("BBBB");
+***REMOVED***
+***REMOVED******REMOVED***
+***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){
+***REMOVED*** System.out.println("CCCC");***REMOVED***
+***REMOVED***
+***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar");
+***REMOVED*** a.copyFromDmToSVN(aa, "Dev");
+
+***REMOVED***
+
+
+***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org",
+***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548"));
+
+***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
+ ***REMOVED*** "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"));
+
+System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org",
+ "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"));
+
+
+***REMOVED***Algorithm aa = new Algorithm();
+***REMOVED***aa.setName("UDPIPE_WRAPPER");
+***REMOVED***System.out.println(a.algoExists(aa));
+***REMOVED******REMOVED***
+***REMOVED***ServiceConfiguration bp = new ServiceConfiguration();
+***REMOVED******REMOVED***
+***REMOVED***SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
+***REMOVED******REMOVED***
+***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); {
+***REMOVED***System.out.println("ciao");
+***REMOVED***
+***REMOVED******REMOVED***
+
+***REMOVED***
+***REMOVED***Algorithm al = new Algorithm();
+***REMOVED*** al.setName("UDPIPE_WRAPPER");
+***REMOVED*** a.deleteFiles(al);
+
+
+
+
+
+***REMOVED***
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java
new file mode 100755
index 0000000..7ceb380
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java
@@ -0,0 +1,288 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
+
+import java.io.File;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+***REMOVED***
+***REMOVED***
+
+import com.jcraft.jsch.Channel;
+import com.jcraft.jsch.ChannelSftp;
+import com.jcraft.jsch.JSch;
+import com.jcraft.jsch.Session;
+import com.jcraft.jsch.SftpException;
+
+public class CheckMethodStaging extends CheckMethod{
+
+***REMOVED***
+
+ public CheckMethodStaging()
+ {
+ this.logger = LoggerFactory.getLogger(CheckMethodStaging.class);
+***REMOVED***
+
+
+
+ @Override
+ public boolean algoExists(Algorithm algo/*, String env*/) throws Exception{
+
+ this.logger.debug("Looking if algo "+algo.getName()+ " exists");
+ Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
+ File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
+ File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
+ this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath());
+
+ System.out.println("First file is located to: "+file.getPath());
+ System.out.println("Second file is located to: "+file2.getPath());
+
+
+ if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))
+ {
+ this.logger.debug("Files found");
+ this.copyFromDmToSVN(file/*,env*/);
+ this.copyFromDmToSVN(file2/*,env*/);
+ System.out.println("Files have been copied to SVN");
+
+
+ return true;
+ ***REMOVED***
+ else
+ {
+ this.logger.debug("Files not found");
+ System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
+ return false;
+ ***REMOVED***
+
+***REMOVED***
+
+
+
+ @Override
+ public void deleteFiles(Algorithm a/*,String env*/) throws Exception{
+ JSch jsch = new JSch();
+ Session session = null;
+ Channel channel = null;
+ ChannelSftp c = null;
+ Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
+ System.out.println("checking existing in env: " + stagingConfiguration.getHost());
+
+ File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
+ File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
+
+
+ System.out.println("First file is located to: "+file.getPath());
+ System.out.println("Second file is located to: "+file2.getPath());
+
+
+ jsch.setKnownHosts("~/.ssh/known_hosts");
+ String privateKey = "~/.ssh/id_rsa";
+
+ jsch.addIdentity(privateKey);
+ System.out.println("Private Key Added.");
+
+ session = jsch.getSession("root", stagingConfiguration.getHost());
+ System.out.println("session created.");
+
+ java.util.Properties config = new java.util.Properties();
+ config.put("StrictHostKeyChecking", "no");
+
+ session.setConfig(config);
+ session.connect();
+
+ channel = session.openChannel("sftp");
+ channel.connect();
+ System.out.println("shell channel connected....");
+
+ c = (ChannelSftp) channel;
+
+ if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){
+
+ c.rm(file.getPath());
+ c.rm(file2.getPath());
+ System.out.println("Both the files have been deleted");
+ ***REMOVED***
+ else System.out.println("Files not found");
+ channel.disconnect();
+ c.disconnect();
+ session.disconnect();
+
+***REMOVED***
+
+
+ @Override
+ public boolean doesExist(String path/*, String env*/) throws Exception {
+ this.logger.debug("Looking if file "+path + " exists");
+ JSch jsch = new JSch();
+ Session session = null;
+ Channel channel = null;
+ ChannelSftp c = null;
+ boolean success = false;
+ Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
+ jsch.setKnownHosts("~/.ssh/known_hosts");
+ String privateKey = "~/.ssh/id_rsa";
+
+ jsch.addIdentity(privateKey);
+ System.out.println("Private Key Added.");
+
+ this.logger.debug("Staging configuration host "+stagingConfiguration.getHost());
+ session = jsch.getSession("root",stagingConfiguration.getHost() );
+ System.out.println("session created.");
+
+ java.util.Properties config = new java.util.Properties();
+ config.put("StrictHostKeyChecking", "no");
+
+ session.setConfig(config);
+ session.connect();
+
+ channel = session.openChannel("sftp");
+ channel.connect();
+ System.out.println("shell channel connected....");
+
+ c = (ChannelSftp) channel;
+
+ ***REMOVED***SftpATTRS is = null;
+ System.out.println(path);
+
+ try {
+ c.lstat(path);
+ this.logger.debug("File found");
+ success = true;
+ ***REMOVED*** catch (SftpException e)
+ {
+ this.logger.error("File not found",e);
+
+ if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE)
+ {
+ ***REMOVED*** file doesn't exist
+ success = false;
+ ***REMOVED***
+ ***REMOVED***success = true; ***REMOVED*** something else went wrong
+ ***REMOVED***
+ channel.disconnect();
+ c.disconnect();
+ session.disconnect();
+ this.logger.debug("Operation result "+success);
+ return success;
+
+***REMOVED***
+
+
+ @Override
+ public void copyFromDmToSVN(File a/*,String env*/) throws Exception {
+ JSch jsch = new JSch();
+ Session session = null;
+ SVNUpdater svnUpdater = new SVNUpdaterStaging();
+ Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
+
+
+ jsch.setKnownHosts("~/.ssh/known_hosts");
+ String privateKey = "~/.ssh/id_rsa";
+
+ jsch.addIdentity(privateKey);
+ System.out.println("Private Key Added.");
+
+ session = jsch.getSession("root", stagingConfiguration.getHost());
+ System.out.println("session created.");
+
+ java.util.Properties config = new java.util.Properties();
+ config.put("StrictHostKeyChecking", "no");
+
+ session.setConfig(config);
+ session.connect();
+
+ Channel channel = session.openChannel("sftp");
+ channel.connect();
+
+ ChannelSftp sftp = (ChannelSftp) channel;
+
+ sftp.cd(stagingConfiguration.getGhostAlgoDirectory());
+
+ System.out.println("REMOTE : "+stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
+ System.out.println("LOCAL : /tmp/"+a.getName());
+
+ sftp.get(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
+
+ channel.disconnect();
+ session.disconnect();
+
+ File f = new File("/tmp/"+a.getName());
+ svnUpdater.updateAlgorithmFiles(f);
+ f.delete();
+***REMOVED***
+
+
+
+
+
+ public static void main(String[] args) throws Exception {
+***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
+***REMOVED*** System.out.println(a.getStagingHost());
+
+ CheckMethodStaging a = new CheckMethodStaging();
+
+ ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
+
+***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar");
+***REMOVED*** System.out.println(aa.getName());
+***REMOVED*** System.out.println(aa.getPath());
+
+
+
+
+
+
+ ***REMOVED***a.copyFromDmToSVN(aa);
+***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")){
+***REMOVED*** System.out.println("AAA");***REMOVED***
+***REMOVED***
+***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){
+***REMOVED*** System.out.println("BBBB");
+***REMOVED***
+***REMOVED******REMOVED***
+***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar")){
+***REMOVED*** System.out.println("CCCC");***REMOVED***
+***REMOVED***
+***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar");
+***REMOVED*** a.copyFromDmToSVN(aa, "Dev");
+
+***REMOVED***
+
+
+***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org",
+***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548"));
+
+***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
+ ***REMOVED*** "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"));
+
+System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org",
+ "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"));
+
+
+***REMOVED***Algorithm aa = new Algorithm();
+***REMOVED***aa.setName("UDPIPE_WRAPPER");
+***REMOVED***System.out.println(a.algoExists(aa));
+***REMOVED******REMOVED***
+***REMOVED***ServiceConfiguration bp = new ServiceConfiguration();
+***REMOVED******REMOVED***
+***REMOVED***SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
+***REMOVED******REMOVED***
+***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); {
+***REMOVED***System.out.println("ciao");
+***REMOVED***
+***REMOVED******REMOVED***
+
+***REMOVED***
+***REMOVED***Algorithm al = new Algorithm();
+***REMOVED*** al.setName("UDPIPE_WRAPPER");
+***REMOVED*** a.deleteFiles(al);
+
+
+
+
+
+***REMOVED***
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java
new file mode 100755
index 0000000..e24d470
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java
@@ -0,0 +1,14 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
+
+public class ClusterBuilderProduction extends ClusterBuilder{
+
+ public ClusterBuilderProduction() {
+ super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration());
+
+***REMOVED***
+
+
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java
new file mode 100755
index 0000000..2fafead
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java
@@ -0,0 +1,13 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
+
+public class ClusterBuilderStaging extends ClusterBuilder{
+
+ public ClusterBuilderStaging() {
+ super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration());
+***REMOVED***
+
+
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java
new file mode 100755
index 0000000..b29311e
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java
@@ -0,0 +1,30 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
+
+***REMOVED***import scala.actors.threadpool.Arrays;
+
+public class NotificationHelperProduction extends NotificationHelper{
+
+***REMOVED*** private Exception executionException;
+
+
+ private String getSubjectHeader() {
+ return "[DataMinerGhostProductionInstallationRequestReport]";
+ ***REMOVED***
+
+ @Override
+ public String getSuccessSubject() {
+ return this.getSubjectHeader()+" is SUCCESS";
+ ***REMOVED***
+
+
+
+ @Override
+ public String getFailedSubject() {
+ return String.format(this.getSubjectHeader()+" is FAILED");
+ ***REMOVED***
+
+
+
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java
new file mode 100755
index 0000000..eace46a
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java
@@ -0,0 +1,32 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
+
+***REMOVED***import scala.actors.threadpool.Arrays;
+
+public class NotificationHelperStaging extends NotificationHelper {
+
+***REMOVED*** private Exception executionException;
+
+
+
+ private String getSubjectHeader() {
+ return "[DataMinerGhostStagingInstallationRequestReport]";
+ ***REMOVED***
+
+
+ @Override
+ public String getSuccessSubject() {
+ return this.getSubjectHeader()+" is SUCCESS";
+ ***REMOVED***
+
+
+
+
+ @Override
+ public String getFailedSubject() {
+ return String.format(this.getSubjectHeader()+" is FAILED");
+ ***REMOVED***
+
+
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java
new file mode 100755
index 0000000..36b4d3a
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java
@@ -0,0 +1,21 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.tmatesoft.svn.core.SVNException;
+
+/**
+ * Created by ggiammat on 5/9/17.
+ */
+public class SVNUpdaterProduction extends SVNUpdater{
+
+
+
+
+ public SVNUpdaterProduction() throws SVNException {
+ super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration());
+***REMOVED***
+
+
+
+***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java
new file mode 100755
index 0000000..eaa6cbf
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java
@@ -0,0 +1,21 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.tmatesoft.svn.core.SVNException;
+
+/**
+ * Created by ggiammat on 5/9/17.
+ */
+public class SVNUpdaterStaging extends SVNUpdater{
+
+
+ public SVNUpdaterStaging() throws SVNException {
+ super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration());
+***REMOVED***
+
+
+
+
+
+***REMOVED***
diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java
index 82c3ee6..8ce58f2 100644
--- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java
+++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java
@@ -6,6 +6,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction;
import org.tmatesoft.svn.core.SVNException;
public class JobTest {
@@ -14,7 +15,7 @@ public class JobTest {
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
- SVNUpdater svnUpdater = new SVNUpdater();
+ SVNUpdater svnUpdater = new SVNUpdaterProduction();
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0");
***REMOVED***test phase
diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java
index 10082d0..779c489 100644
--- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java
+++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java
@@ -5,6 +5,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging;
import org.tmatesoft.svn.core.SVNException;
/**
@@ -17,7 +18,7 @@ public class SVNTests {
public static void main(String[] args) throws SVNException, IOException, InterruptedException {
- SVNUpdater svnUpdater = new SVNUpdater();
+ SVNUpdater svnUpdater = new SVNUpdaterStaging();
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0");
***REMOVED***algo.setClazz(algo.getClazz() + "TEST");
@@ -27,7 +28,7 @@ public class SVNTests {
***REMOVED***System.out.println(algo.getCategory());
***REMOVED***System.out.println(algo.getSkipJava());
- svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), "");
+ ***REMOVED***svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), "");
***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto");
***REMOVED***svnUpdater.readRPRotoDeps(algo);