gCubeSystem
/
dataminer-pool-manager
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@158895 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Ciro Formisano 2017-11-29 15:29:01 +00:00
parent f696d7af4c
commit 325f3a3b86
25 changed files with 1087 additions and 1002 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataAnalysis</groupId> <groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId> <artifactId>dataminer-pool-manager</artifactId>
<packaging>war</packaging> <packaging>war</packaging>
<version>2.2.0-SNAPSHOT</version> <version>2.3.0-SNAPSHOT</version>
<name>dataminer-pool-manager</name> <name>dataminer-pool-manager</name>
<description> <description>

44
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
View File

@ -1,41 +1,16 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible; package org.gcube.dataanalysis.dataminer.poolmanager.ansible;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
***REMOVED*** ***REMOVED***
import java.io.InputStream; import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream; import java.io.PrintStream;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
***REMOVED***
import java.util.Scanner; import java.util.Scanner;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
import org.tmatesoft.svn.core.io.ISVNEditor;
import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
/** /**
* This class is responsible for the interface with ansible, retrieving log, * This class is responsible for the interface with ansible, retrieving log,
@ -72,12 +47,12 @@ public class AnsibleWorker ***REMOVED***
this.ensureWorkStructure(); this.ensureWorkStructure();
***REMOVED*** ***REMOVED***
public File getWorkdir() ***REMOVED*** ***REMOVED*** public File getWorkdir() ***REMOVED***
return this.workerRoot; ***REMOVED*** return this.workerRoot;
***REMOVED*** ***REMOVED*** ***REMOVED***
public File getRolesDir() ***REMOVED*** public File getRolesDir() ***REMOVED***
return new File(this.getWorkdir(), ROLES_DIR); return new File(this.workerRoot, ROLES_DIR);
***REMOVED*** ***REMOVED***
public String getWorkerId() ***REMOVED*** public String getWorkerId() ***REMOVED***
@ -86,20 +61,20 @@ public class AnsibleWorker ***REMOVED***
public void ensureWorkStructure() ***REMOVED*** public void ensureWorkStructure() ***REMOVED***
***REMOVED*** generate root ***REMOVED*** generate root
this.getWorkdir().mkdirs(); this.workerRoot.mkdirs();
***REMOVED*** ***REMOVED***
public void removeWorkStructure() ***REMOVED*** public void removeWorkStructure() ***REMOVED***
***REMOVED*** remove the working dir ***REMOVED*** remove the working dir
this.getWorkdir().delete(); this.workerRoot.delete();
***REMOVED*** ***REMOVED***
public File getPlaybookFile() ***REMOVED*** public File getPlaybookFile() ***REMOVED***
return new File(this.getWorkdir(), PLAYBOOK_NAME); return new File(this.workerRoot, PLAYBOOK_NAME);
***REMOVED*** ***REMOVED***
public File getInventoryFile() ***REMOVED*** public File getInventoryFile() ***REMOVED***
return new File(this.getWorkdir(), INVENTORY_NAME); return new File(this.workerRoot, INVENTORY_NAME);
***REMOVED*** ***REMOVED***
@ -122,7 +97,7 @@ public class AnsibleWorker ***REMOVED***
public int execute(PrintStream ps) public int execute(PrintStream ps)
throws IOException, InterruptedException, SVNException ***REMOVED*** throws IOException, InterruptedException, SVNException ***REMOVED***
System.out.println(this.getWorkdir()); System.out.println(this.workerRoot);
try ***REMOVED*** try ***REMOVED***
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath()
+ " " + this.getPlaybookFile().getAbsolutePath()); + " " + this.getPlaybookFile().getAbsolutePath());
@ -149,6 +124,7 @@ public class AnsibleWorker ***REMOVED***
while (sc.hasNextLine()) ***REMOVED*** while (sc.hasNextLine()) ***REMOVED***
dest.println(sc.nextLine()); dest.println(sc.nextLine());
***REMOVED*** ***REMOVED***
sc.close();
***REMOVED*** ***REMOVED***
***REMOVED***).start(); ***REMOVED***).start();
***REMOVED*** ***REMOVED***

17
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -1,10 +1,7 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge;
import java.io.File; import java.io.File;
import java.io.FileOutputStream;
***REMOVED*** ***REMOVED***
import java.io.InputStream;
import java.io.PrintStream;
import java.util.Collection; import java.util.Collection;
***REMOVED*** ***REMOVED***
import java.util.Map; import java.util.Map;
@ -32,21 +29,19 @@ import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
***REMOVED***
***REMOVED***
public class AnsibleBridge ***REMOVED*** public class AnsibleBridge ***REMOVED***
private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); ***REMOVED***private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class);
private String dpmRoot; private String dpmRoot;
public AnsibleBridge() ***REMOVED*** ***REMOVED*** public AnsibleBridge() ***REMOVED***
this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); ***REMOVED*** this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager");
***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager")); ***REMOVED*** ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager"));
***REMOVED***
***REMOVED*** ***REMOVED*** ***REMOVED***
public AnsibleBridge(String root) ***REMOVED*** public AnsibleBridge(String root) ***REMOVED***
this.dpmRoot = root; this.dpmRoot = root;

219
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java
View File

@ -5,34 +5,71 @@ import java.io.FileOutputStream;
***REMOVED*** ***REMOVED***
import java.io.PrintStream; import java.io.PrintStream;
import java.io.PrintWriter; import java.io.PrintWriter;
import java.util.LinkedList; import java.util.Collection;
***REMOVED***
import java.util.UUID; import java.util.UUID;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException;
***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED***
public abstract class DMPMJob ***REMOVED*** public abstract class DMPMJob ***REMOVED***
private Configuration configuration;
private String dmpmHomeDirectory;
private SVNUpdater svnUpdater;
private File jobLogs;
private String id;
private Algorithm algorithm;
private Cluster cluster;
private String vREName;
private String category;
private String algorithm_type;
private Logger logger;
protected SVNUpdater svnUpdater; private enum STATUS
protected File jobLogs; ***REMOVED***
PROGRESS ("IN PROGRESS"),
COMPLETED ("COMPLETED"),
FAILED ("FAILED");
private String status;
STATUS (String status)
***REMOVED***
this.status = status;
***REMOVED***
***REMOVED***
protected String id; public DMPMJob(SVNUpdater svnUpdater,Configuration configuration,Algorithm algorithm, Cluster cluster,String vREName,
String category, String algorithm_type)***REMOVED***
this.logger = LoggerFactory.getLogger(DMPMJob.class);
public DMPMJob(SVNUpdater svnUpdater)***REMOVED*** this.configuration = configuration;
this.algorithm = algorithm;
this.cluster = cluster;
this.vREName = vREName;
this.category = category;
this.algorithm_type = algorithm_type;
this.svnUpdater = svnUpdater; this.svnUpdater = svnUpdater;
this.dmpmHomeDirectory = new String (System.getProperty("user.home")+File.separator+"dataminer-pool-manager");
this.id = UUID.randomUUID().toString(); this.id = UUID.randomUUID().toString();
***REMOVED***TODO: dmpm work directory should be loaded from configuration file ***REMOVED***TODO: dmpm work directory should be loaded from configuration file
this.jobLogs = new File(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"+File.separator+"jobs"); this.jobLogs = new File(this.dmpmHomeDirectory+File.separator+"jobs");
this.jobLogs.mkdirs(); this.jobLogs.mkdirs();
***REMOVED*** ***REMOVED***
@ -56,7 +93,7 @@ public abstract class DMPMJob ***REMOVED***
Cluster dataminerCluster, Cluster dataminerCluster,
boolean includeAlgorithmDependencies, boolean includeAlgorithmDependencies,
String user)***REMOVED*** String user)***REMOVED***
AnsibleBridge ansibleBridge = new AnsibleBridge(); AnsibleBridge ansibleBridge = new AnsibleBridge(this.dmpmHomeDirectory);
try ***REMOVED*** try ***REMOVED***
return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user);
***REMOVED*** catch (IOException e) ***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED***
@ -66,14 +103,148 @@ public abstract class DMPMJob ***REMOVED***
***REMOVED*** ***REMOVED***
protected abstract void execute(); public void setStatusInformation(STATUS exitStatus) ***REMOVED***
protected int executeAnsibleWorker(AnsibleWorker worker) throws IOException, InterruptedException, SVNException***REMOVED***
File path = new File(worker.getWorkdir() + File.separator + "jobs");
path.mkdirs();
File n = new File(this.jobLogs + File.separator + this.id);
FileOutputStream fos = new FileOutputStream(n, true); try
***REMOVED***
File statusFile = new File (this.jobLogs,this.id + "_exitStatus");
***REMOVED***File m = new File ( this.jobLogs + File.separator + this.id + "_exitStatus");
PrintWriter writer = new PrintWriter(statusFile, "UTF-8");
writer.println(exitStatus.status);
writer.close();
***REMOVED*** catch (Exception e)
***REMOVED***
this.logger.error ("Unable to update exit status file with status "+exitStatus.status,e);
***REMOVED***
***REMOVED***
private void updateLogFile (File logFile, String message)
***REMOVED***
try
***REMOVED***
PrintWriter writer = new PrintWriter(logFile,"UTF-8");
writer.print(message);
writer.close();
***REMOVED*** catch (Exception e)
***REMOVED***
this.logger.error("Unable to log the error message: "+message,e);
***REMOVED***
***REMOVED***
protected abstract void execute ();
private boolean preInstallation (SendMail sm,NotificationHelper nh, File logFile ) throws SVNException, EMailException
***REMOVED***
this.logger.debug("Checking dependencies...");
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
this.svnUpdater.getDependencyFile(this.algorithm.getLanguage()),
this.algorithm.getDependencies());
if (!undefinedDependencies.isEmpty())
***REMOVED***
this.logger.debug("Some dependencies are not defined");
String message = "Following dependencies are not defined:\n";
for (String n : undefinedDependencies) ***REMOVED***
message += "\n" + n +"\n";
***REMOVED***
this.setStatusInformation(STATUS.FAILED);
String errorMessage = nh.getFailedBody(message+"\n\n"+this.buildInfo());
this.updateLogFile(logFile, errorMessage);
sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", errorMessage);
return false;
***REMOVED***
else return true;
***REMOVED***
private void installation (SendMail sm,NotificationHelper nh,CheckMethod methodChecker,File logFile ) throws Exception
***REMOVED***
methodChecker.deleteFiles(this.algorithm/*, env*/);
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.cluster, false, "root"),logFile);
System.out.println("Return code= "+ret);
if (ret != 0)
***REMOVED***
this.logger.debug("Ansible work failed, return code "+ret);
this.setStatusInformation(STATUS.FAILED);
String errorMessage = nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo();
sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",errorMessage);
***REMOVED***
else if (ret == 0)
***REMOVED***
this.logger.debug("Operation completed");
this.setStatusInformation(STATUS.PROGRESS);
this.logger.debug("Checking the method...");
if (methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get())&&(methodChecker.algoExists(this.algorithm)))
***REMOVED***
this.logger.debug("Method OK and algo exists");
System.out.println("Interface check ok!");
System.out.println("Both the files exist at the correct path!");
this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.vREName,this.category, this.algorithm_type,
this.algorithm.getFullname());
this.setStatusInformation(STATUS.COMPLETED);
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
return;
***REMOVED*** else
***REMOVED***
this.logger.debug("Operation failed");
this.setStatusInformation(STATUS.FAILED);
sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",
nh.getFailedBody(
"\n"+
"Installation completed but DataMiner Interface not working correctly or files "
+ this.algorithm.getName() + ".jar and " + this.algorithm.getName()
+ "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
***REMOVED***
***REMOVED***
***REMOVED***
protected void execute(NotificationHelper nh, CheckMethod methodChecker)
***REMOVED***
SendMail sm = new SendMail();
try ***REMOVED***
this.logger.debug("Pre installation operations");
File logFile = new File(this.jobLogs,this.id);
***REMOVED***File logFile = new File(this.jobLogs + File.separator + this.id);
boolean preInstallationResponse = preInstallation(sm, nh, logFile);
this.logger.debug("Pre installation operation completed with result "+preInstallationResponse);
if (preInstallationResponse)
***REMOVED***
this.logger.debug("Installation...");
installation(sm, nh, methodChecker, logFile);
this.logger.debug("Installation completed");
***REMOVED***
***REMOVED*** catch (EMailException eme)
***REMOVED***
this.logger.error("Operation failed and unable to send notification email",eme);
***REMOVED***
catch (Exception e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***
protected int executeAnsibleWorker(AnsibleWorker worker, File logFile) throws IOException, InterruptedException, SVNException***REMOVED***
FileOutputStream fos = new FileOutputStream(logFile, true);
PrintStream ps = new PrintStream(fos); PrintStream ps = new PrintStream(fos);
***REMOVED*** File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); ***REMOVED*** File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
@ -81,5 +252,19 @@ public abstract class DMPMJob ***REMOVED***
return worker.execute(ps); return worker.execute(ps);
***REMOVED*** ***REMOVED***
public String buildInfo() ***REMOVED***
return
"\n"+
"Algorithm details:\n"+"\n"+
"User: "+this.algorithm.getFullname()+"\n"+
"Algorithm name: "+this.algorithm.getName()+"\n"+
"Staging DataMiner Host: "+ this.configuration.getHost()+"\n"+
"Caller VRE: "+ScopeProvider.instance.get()+"\n"+
"Target VRE: "+this.vREName+"\n";
***REMOVED***
***REMOVED*** ***REMOVED***

22
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -8,7 +8,10 @@ import java.util.Scanner;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderProduction;
import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderStaging;
import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction;
import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
@ -17,14 +20,15 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
public class DataminerPoolManager ***REMOVED*** public class DataminerPoolManager ***REMOVED***
private Logger logger; private Logger logger;
private SVNUpdaterStaging svnUpdaterStaging;
private SVNUpdater svnUpdater; private SVNUpdaterProduction svnUpdaterProduction;
public DataminerPoolManager() ***REMOVED*** public DataminerPoolManager() ***REMOVED***
this.logger = LoggerFactory.getLogger(this.getClass()); this.logger = LoggerFactory.getLogger(this.getClass());
try ***REMOVED*** try ***REMOVED***
***REMOVED***TODO: read this from configuration ***REMOVED***TODO: read this from configuration
this.svnUpdater = new SVNUpdater(); this.svnUpdaterStaging = new SVNUpdaterStaging();
this.svnUpdaterProduction = new SVNUpdaterProduction();
***REMOVED*** catch (SVNException e) ***REMOVED*** ***REMOVED*** catch (SVNException e) ***REMOVED***
e.printStackTrace(); e.printStackTrace();
***REMOVED*** ***REMOVED***
@ -37,9 +41,10 @@ public class DataminerPoolManager ***REMOVED***
this.logger.debug("Algo "+algo); this.logger.debug("Algo "+algo);
this.logger.debug("Category "+category); this.logger.debug("Category "+category);
this.logger.debug("Algo type "+algorithm_type); this.logger.debug("Algo type "+algorithm_type);
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); ClusterBuilder stagingClusterBuilder = new ClusterBuilderStaging();
Cluster stagingCluster = stagingClusterBuilder.getDataminerCluster();
***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); ***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/); DMPMJob job = new StagingJob(this.svnUpdaterStaging, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/);
String id = job.start(); String id = job.start();
return id; return id;
***REMOVED*** ***REMOVED***
@ -50,8 +55,9 @@ public class DataminerPoolManager ***REMOVED***
this.logger.debug("Algo "+algo); this.logger.debug("Algo "+algo);
this.logger.debug("Category "+category); this.logger.debug("Category "+category);
this.logger.debug("Algo type "+algorithm_type); this.logger.debug("Algo type "+algorithm_type);
Cluster prodCluster = ClusterBuilder.getProdDataminerCluster(); ClusterBuilder productionClusterBuilder = new ClusterBuilderProduction();
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/); Cluster prodCluster = productionClusterBuilder.getDataminerCluster();
DMPMJob job = new ProductionPublishingJob(this.svnUpdaterProduction, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/);
String id = job.start(); String id = job.start();
return id; return id;
***REMOVED*** ***REMOVED***

144
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
View File

@ -1,164 +1,44 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service; package org.gcube.dataanalysis.dataminer.poolmanager.service;
import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.Collection;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodProduction;
import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperProduction;
***REMOVED***
***REMOVED***
public class ProductionPublishingJob extends DMPMJob ***REMOVED*** public class ProductionPublishingJob extends DMPMJob ***REMOVED***
private Algorithm algorithm;
private Cluster prodCluster;
private String targetVREName;
private String category;
private String algorithm_type;
***REMOVED***private String targetVREToken; ***REMOVED***private String targetVREToken;
***REMOVED***private String env; ***REMOVED***private String env;
private Logger logger;
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm,
Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED***
super(svnUpdater); super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(),algorithm,prodCluster,targetVREName,category,algorithm_type);
this.jobLogs = new File( this.logger = LoggerFactory.getLogger(StagingJob.class);***REMOVED*** this.jobLogs = new File(
System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
this.jobLogs.mkdirs();
this.algorithm = algorithm;
this.prodCluster = prodCluster;
this.targetVREName = targetVREName;
this.category = category;
this.algorithm_type = algorithm_type;
***REMOVED***this.targetVREToken = targetVREToken;
***REMOVED***this.env= env;
this.getStatus(0);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
protected void execute() ***REMOVED*** protected void execute() ***REMOVED***
SendMail sm = new SendMail(); this.logger.debug("Executing staging job...");
NotificationHelper nh = new NotificationHelper(); super.execute(new NotificationHelperProduction(), new CheckMethodProduction());
CheckMethod b = new CheckMethod();
try ***REMOVED***
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
this.svnUpdater.getDependencyFileProd(this.algorithm.getLanguage()/*,env*/),
this.algorithm.getDependencies());
if (!undefinedDependencies.isEmpty()) ***REMOVED***
String message = "Following dependencies are not defined:\n";
for (String n : undefinedDependencies) ***REMOVED***
message += "\n" + n +"\n";
***REMOVED***
this.getStatus(2);
sm.sendNotification(nh.getFailedSubjectRelease() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo()));
return;
***REMOVED***
b.deleteFilesProd(this.algorithm);
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.prodCluster, false, "root"));
if (ret != 0) ***REMOVED***
this.getStatus(2);
sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo());
return;
***REMOVED***
if (ret == 0) ***REMOVED***
this.getStatus(0);
if (b.checkMethod(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost(), SecurityTokenProvider.instance.get())&&(b.algoExistsProd(this.algorithm/*, env*/))) ***REMOVED***
System.out.println("Interface check ok!");
System.out.println("Both the files exist at the correct path!");
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName,this.category, this.algorithm_type,
this.algorithm.getFullname()/*, env*/);
this.getStatus(9);
sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
return;
***REMOVED*** else
this.getStatus(2);
sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm",
nh.getFailedBody(
"\n"+
"Installation completed but DataMiner Interface not working correctly or files "
+ this.algorithm.getName() + ".jar and " + this.algorithm.getName()
+ "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
return;
***REMOVED***
***REMOVED*** catch (Exception e) ***REMOVED***
try ***REMOVED***
this.getStatus(0);
***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED*** catch (UnsupportedEncodingException e1) ***REMOVED***
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED*** ***REMOVED***
public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED***
File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
PrintWriter writer = new PrintWriter(m, "UTF-8");
String response = "";
if (exitstatus == 0) ***REMOVED***
response = "IN PROGRESS";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
if (exitstatus == 9) ***REMOVED***
response = "COMPLETED";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
if (exitstatus == 2) ***REMOVED***
response = "FAILED";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
writer.close();
return response;
***REMOVED***
public String buildInfo() throws UnsupportedEncodingException***REMOVED***
return
"\n"+
"Algorithm details:\n"+"\n"+
"User: "+this.algorithm.getFullname()+"\n"+
"Algorithm name: "+this.algorithm.getName()+"\n"+
"Staging DataMiner Host: "+ DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost()+"\n"+
"Caller VRE: "+ScopeProvider.instance.get()+"\n"+
"Target VRE: "+this.targetVREName+"\n";
***REMOVED***

152
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
View File

@ -1,184 +1,42 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service; package org.gcube.dataanalysis.dataminer.poolmanager.service;
import java.io.File;
import java.io.FileNotFoundException; import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException; import java.io.UnsupportedEncodingException;
import java.util.Collection;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodStaging;
import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperStaging;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
public class StagingJob extends DMPMJob ***REMOVED*** public class StagingJob extends DMPMJob ***REMOVED***
private Algorithm algorithm;
private Cluster stagingCluster;
***REMOVED*** private Cluster rProtoCluster;
private String rProtoVREName;
***REMOVED***private String env;
private String category;
private String algorithm_type;
private Logger logger; private Logger logger;
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm,
Cluster stagingCluster, /* Cluster rProtoCluster, */ Cluster stagingCluster, /* Cluster rProtoCluster, */
String rProtoVREName, String category, String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** String rProtoVREName, String category, String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED***
super(svnUpdater); super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(),algorithm,stagingCluster,rProtoVREName,category,algorithm_type);
this.logger = LoggerFactory.getLogger(StagingJob.class); this.logger = LoggerFactory.getLogger(StagingJob.class);
this.jobLogs = new File(
System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
this.jobLogs.mkdirs();
this.algorithm = algorithm;
this.stagingCluster = stagingCluster;
***REMOVED*** this.rProtoCluster = rProtoCluster;
this.rProtoVREName = rProtoVREName;
***REMOVED***this.env = env;
this.category = category;
this.algorithm_type = algorithm_type;
***REMOVED***File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
***REMOVED***PrintWriter writer = new PrintWriter(m, "UTF-8");
this.getStatus(0);
***REMOVED***writer.close();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
protected void execute() ***REMOVED*** protected void execute() ***REMOVED***
this.logger.debug("Executing staging job..."); this.logger.debug("Executing staging job...");
super.execute(new NotificationHelperStaging(), new CheckMethodStaging());
CheckMethod methodChecker = new CheckMethod();
SendMail sm = new SendMail();
NotificationHelper nh = new NotificationHelper();
try ***REMOVED***
this.logger.debug("Checking dependencies...");
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
this.svnUpdater.getDependencyFile(this.algorithm.getLanguage()/*,env*/),
this.algorithm.getDependencies());
if (!undefinedDependencies.isEmpty())
***REMOVED***
this.logger.debug("Some dependencies are not defined");
String message = "Following dependencies are not defined:\n";
for (String n : undefinedDependencies) ***REMOVED***
message += "\n" + n +"\n";
***REMOVED***
this.getStatus(2);
sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo()));
return;
***REMOVED***
***REMOVED***before the installation to check if the files exist
methodChecker.deleteFiles(this.algorithm/*, env*/);
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root"));
System.out.println("Return code= "+ret);
if (ret != 0)
***REMOVED***
this.logger.debug("Ansible work failed, return code "+ret);
this.getStatus(2);
sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo());
return;
***REMOVED***
if (ret == 0)
***REMOVED***
this.logger.debug("Operation completed");
this.getStatus(0);
***REMOVED***System.out.println("1 - Checking existing in env: "+ env);
***REMOVED***System.out.println("2 - Checking existing in env: "+ this.env);
this.logger.debug("Checking the method...");
if (methodChecker.checkMethod(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost(), SecurityTokenProvider.instance.get())&&(methodChecker.algoExists(this.algorithm)))
***REMOVED***
this.logger.debug("Method OK and algo exists");
System.out.println("Interface check ok!");
System.out.println("Both the files exist at the correct path!");
this.svnUpdater.updateSVNStagingAlgorithmList(this.algorithm, this.rProtoVREName,this.category, this.algorithm_type,
this.algorithm.getFullname()/*, env*/);
this.getStatus(9);
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
return;
***REMOVED*** else
***REMOVED***
this.logger.debug("Operation failed");
this.getStatus(2);
sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",
nh.getFailedBody(
"\n"+
"Installation completed but DataMiner Interface not working correctly or files "
+ this.algorithm.getName() + ".jar and " + this.algorithm.getName()
+ "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
return;
***REMOVED***
***REMOVED***
***REMOVED*** catch (Exception e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED*** ***REMOVED***
public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED***
File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
PrintWriter writer = new PrintWriter(m, "UTF-8");
String response = "";
if (exitstatus == 0) ***REMOVED***
response = "IN PROGRESS";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
if (exitstatus == 9) ***REMOVED***
response = "COMPLETED";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
if (exitstatus == 2) ***REMOVED***
response = "FAILED";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
writer.close();
return response;
***REMOVED***
public String buildInfo() throws UnsupportedEncodingException***REMOVED***
return
"\n"+
"Algorithm details:\n"+"\n"+
"User: "+this.algorithm.getFullname()+"\n"+
"Algorithm name: "+this.algorithm.getName()+"\n"+
"Staging DataMiner Host: "+ DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost()+"\n"+
"Caller VRE: "+ScopeProvider.instance.get()+"\n"+
"Target VRE: "+rProtoVREName+"\n";
***REMOVED***

436
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java
View File

@ -10,20 +10,11 @@ import java.util.Arrays;
import java.util.LinkedList; import java.util.LinkedList;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
import com.jcraft.jsch.Channel; public abstract class CheckMethod ***REMOVED***
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpATTRS;
import com.jcraft.jsch.SftpException;
public class CheckMethod ***REMOVED***
private Logger logger; private Logger logger;
@ -37,14 +28,14 @@ public class CheckMethod ***REMOVED***
this.logger.debug("Checking method for machine "+machine); this.logger.debug("Checking method for machine "+machine);
this.logger.debug("By using tocken "+token); this.logger.debug("By using tocken "+token);
System.out.println("Machine: " + machine); System.out.println("Machine: " + machine);
String getCapabilitesRequest = new String(); ***REMOVED*** String getCapabilitesRequest = new String();
String getCapabilitesResponse = new String(); ***REMOVED*** String getCapabilitesResponse = new String();
System.out.println(" Token: " + token); System.out.println(" Token: " + token);
String request = "http:***REMOVED***" + machine String request = "http:***REMOVED***" + machine
+ "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token; + "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token;
String response = machine + "___" + token + ".xml"; String response = machine + "___" + token + ".xml";
getCapabilitesRequest = request; ***REMOVED*** getCapabilitesRequest = request;
getCapabilitesResponse = response; ***REMOVED*** getCapabilitesResponse = response;
String baseDescriptionRequest = "http:***REMOVED***" + machine String baseDescriptionRequest = "http:***REMOVED***" + machine
+ "/wps/WebProcessingService?Request=DescribeProcess&Service=WPS&Version=1.0.0" + "&gcube-token=" + "/wps/WebProcessingService?Request=DescribeProcess&Service=WPS&Version=1.0.0" + "&gcube-token="
+ token + "&Identifier="; + token + "&Identifier=";
@ -111,359 +102,22 @@ public class CheckMethod ***REMOVED***
public boolean algoExists(Algorithm algo/*, String env*/) throws Exception***REMOVED*** public abstract boolean algoExists(Algorithm algo/*, String env*/) throws Exception;
this.logger.debug("Looking if algo "+algo.getName()+ " exists");
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath());
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))
***REMOVED***
this.logger.debug("Files found");
this.copyFromDmToSVN(file/*,env*/);
this.copyFromDmToSVN(file2/*,env*/);
System.out.println("Files have been copied to SVN");
return true;
***REMOVED***
else
***REMOVED***
this.logger.debug("Files not found");
System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
return false;
***REMOVED***
***REMOVED***
public abstract void deleteFiles(Algorithm a/*,String env*/) throws Exception;
public boolean algoExistsProd(Algorithm a/*, String env*/) throws Exception***REMOVED*** public abstract boolean doesExist(String path/*, String env*/) throws Exception;
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
if ((this.doesExistProd(file.getPath()/*,env*/)) && (this.doesExistProd(file2.getPath()/*,env*/)))***REMOVED***
this.copyFromDmToSVNProd(file/*,env*/);
this.copyFromDmToSVNProd(file2/*,env*/);
return true;
***REMOVED***
else
System.out.println("Algorithm "+a.getName()+".jar"+ " and "+a.getName()+"_interface.jar files are not present at the expected path");
return false;
***REMOVED***
public void deleteFiles(Algorithm a/*,String env*/) throws Exception***REMOVED*** public abstract void copyFromDmToSVN(File a/*,String env*/) throws Exception;
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
System.out.println("checking existing in env: " + stagingConfiguration.getHost());
File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", stagingConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/)))***REMOVED***
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
***REMOVED***
else System.out.println("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
***REMOVED***
public void deleteFilesProd(Algorithm a/*,String env*/) throws Exception***REMOVED***
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
System.out.println("checking existing in env: " + productionConfiguration.getHost());
File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
if(doesExistProd(file.getPath()/*,env*/)&&(doesExistProd(file2.getPath()/*,env*/)))***REMOVED***
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
***REMOVED***
else System.out.println("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
***REMOVED***
public boolean doesExist(String path/*, String env*/) throws Exception ***REMOVED***
this.logger.debug("Looking if file "+path + " exists");
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
boolean success = false;
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
this.logger.debug("Staging configuration host "+stagingConfiguration.getHost());
session = jsch.getSession("root",stagingConfiguration.getHost() );
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
SftpATTRS is = null;
System.out.println(path);
try ***REMOVED***
is = c.lstat(path);
this.logger.debug("File found");
success = true;
***REMOVED*** catch (SftpException e)
***REMOVED***
this.logger.error("File not found",e);
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE)
***REMOVED***
***REMOVED*** file doesn't exist
success = false;
***REMOVED***
***REMOVED***success = true; ***REMOVED*** something else went wrong
***REMOVED***
channel.disconnect();
c.disconnect();
session.disconnect();
this.logger.debug("Operation result "+success);
return success;
***REMOVED***
public boolean doesExistProd(String path/*, String env*/) throws Exception ***REMOVED***
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
boolean success = false;
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
SftpATTRS is = null;
System.out.println(path);
try ***REMOVED***
is = c.lstat(path);
success = true;
***REMOVED*** catch (SftpException e) ***REMOVED***
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) ***REMOVED***
***REMOVED*** file doesn't exist
success = false;
***REMOVED***
***REMOVED***success = true; ***REMOVED*** something else went wrong
***REMOVED***
channel.disconnect();
c.disconnect();
session.disconnect();
return success;
***REMOVED***
public void copyFromDmToSVN(File a/*,String env*/) throws Exception ***REMOVED***
JSch jsch = new JSch();
Session session = null;
SVNUpdater svnUpdater = new SVNUpdater();
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", stagingConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
ChannelSftp sftp = (ChannelSftp) channel;
sftp.cd(stagingConfiguration.getGhostAlgoDirectory());
System.out.println("REMOTE : "+stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
System.out.println("LOCAL : /tmp/"+a.getName());
sftp.get(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
channel.disconnect();
session.disconnect();
File f = new File("/tmp/"+a.getName());
svnUpdater.updateAlgorithmFiles(f);
f.delete();
***REMOVED***
public void copyFromDmToSVNProd(File a/*,String env*/) throws Exception ***REMOVED***
JSch jsch = new JSch();
Session session = null;
SVNUpdater svnUpdater = new SVNUpdater();
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
ChannelSftp sftp = (ChannelSftp) channel;
sftp.cd(productionConfiguration.getGhostAlgoDirectory());
System.out.println("REMOTE : "+productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
System.out.println("LOCAL : /tmp/"+a.getName());
sftp.get(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
channel.disconnect();
session.disconnect();
File f = new File("/tmp/"+a.getName());
svnUpdater.updateAlgorithmFilesProd(f);
f.delete();
***REMOVED***
public List<String> getFiles(String a)***REMOVED*** public static List<String> getFiles(String a)***REMOVED***
String[] array = a.split(","); String[] array = a.split(",");
ArrayList<String> list = new ArrayList<>(Arrays.asList(array)); ArrayList<String> list = new ArrayList<>(Arrays.asList(array));
@ -477,71 +131,5 @@ public class CheckMethod ***REMOVED***
***REMOVED*** ***REMOVED***
public static void main(String[] args) throws Exception ***REMOVED***
***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
***REMOVED*** System.out.println(a.getStagingHost());
CheckMethod a = new CheckMethod();
***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar");
***REMOVED*** System.out.println(aa.getName());
***REMOVED*** System.out.println(aa.getPath());
***REMOVED***a.copyFromDmToSVN(aa);
***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "***REMOVED***"))***REMOVED***
***REMOVED*** System.out.println("AAA");***REMOVED***
***REMOVED***
***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar"))***REMOVED***
***REMOVED*** System.out.println("BBBB");
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar"))***REMOVED***
***REMOVED*** System.out.println("CCCC");***REMOVED***
***REMOVED***
***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar");
***REMOVED*** a.copyFromDmToSVN(aa, "Dev");
***REMOVED***
***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org",
***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548"));
***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
***REMOVED*** "***REMOVED***"));
System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org",
"***REMOVED***"));
***REMOVED***Algorithm aa = new Algorithm();
***REMOVED***aa.setName("UDPIPE_WRAPPER");
***REMOVED***System.out.println(a.algoExists(aa));
***REMOVED******REMOVED***
***REMOVED***ServiceConfiguration bp = new ServiceConfiguration();
***REMOVED******REMOVED***
***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***");
***REMOVED******REMOVED***
***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); ***REMOVED***
***REMOVED***System.out.println("ciao");
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***Algorithm al = new Algorithm();
***REMOVED*** al.setName("UDPIPE_WRAPPER");
***REMOVED*** a.deleteFiles(al);
***REMOVED***
***REMOVED***

6
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java
View File

@ -1,11 +1,9 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util; package org.gcube.dataanalysis.dataminer.poolmanager.util;
import static org.gcube.common.authorization.client.Constants.authorizationService;
***REMOVED*** ***REMOVED***
import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.AuthorizationEntry;
***REMOVED***
***REMOVED***
import static org.gcube.common.authorization.client.Constants.authorizationService;
public class CheckPermission ***REMOVED*** public class CheckPermission ***REMOVED***

51
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
View File

@ -6,58 +6,37 @@ import java.io.FileNotFoundException;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
public class ClusterBuilder ***REMOVED*** public abstract class ClusterBuilder ***REMOVED***
private Configuration configuration;
public ClusterBuilder (Configuration configuration)
***REMOVED***
this.configuration = configuration;
***REMOVED***
***REMOVED***1. to complete ***REMOVED***1. to complete
public static Cluster getStagingDataminerCluster() throws FileNotFoundException***REMOVED*** public Cluster getDataminerCluster() throws FileNotFoundException***REMOVED***
Cluster cluster = new Cluster(); Cluster cluster = new Cluster();
Host h = new Host(); Host h = new Host();
h.setName(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost()); h.setName(this.configuration.getHost());
cluster.addHost(h); cluster.addHost(h);
***REMOVED*** if (env.equals("Dev"))***REMOVED***
***REMOVED*** h.setName(p.getDevStagingHost());
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto"))))***REMOVED***
***REMOVED*** h.setName(p.getProtoProdStagingHost());
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
return cluster;
***REMOVED***
public static Cluster getProdDataminerCluster() throws FileNotFoundException***REMOVED***
Cluster cluster = new Cluster();
Host h = new Host();
h.setName(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost());
cluster.addHost(h);
***REMOVED*** if (env.equals("Dev"))***REMOVED***
***REMOVED*** h.setName(p.getDevStagingHost());
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto"))))***REMOVED***
***REMOVED*** h.setName(p.getProtoProdStagingHost());
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
return cluster; return cluster;
***REMOVED*** ***REMOVED***
public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException***REMOVED***
public Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException***REMOVED***
Cluster cluster = new Cluster(); Cluster cluster = new Cluster();
for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) ***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) ***REMOVED***
cluster.addHost(h); cluster.addHost(h);
@ -65,13 +44,13 @@ public class ClusterBuilder ***REMOVED***
return cluster; return cluster;
***REMOVED*** ***REMOVED***
public static Cluster getRProtoCluster() throws IOException***REMOVED*** public Cluster getRProtoCluster() throws IOException***REMOVED***
***REMOVED***Assumes the service is running in RPrototypingLab ***REMOVED***Assumes the service is running in RPrototypingLab
String token = SecurityTokenProvider.instance.get(); String token = SecurityTokenProvider.instance.get();
String targetVRE = ScopeProvider.instance.get(); String targetVRE = ScopeProvider.instance.get();
return getVRECluster(token, targetVRE); return this.getVRECluster(token, targetVRE);
***REMOVED*** ***REMOVED***

51
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java
View File

@ -1,53 +1,28 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util; package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import org.apache.commons.lang.StringUtils;
***REMOVED***import scala.actors.threadpool.Arrays; ***REMOVED***import scala.actors.threadpool.Arrays;
public class NotificationHelper ***REMOVED*** public abstract class NotificationHelper ***REMOVED***
private Exception executionException; ***REMOVED*** private Exception executionException;
public NotificationHelper() ***REMOVED***
***REMOVED***
private String getSubjectHeader() ***REMOVED***
return "[DataMinerGhostStagingInstallationRequestReport]";
***REMOVED*** ***REMOVED*** private boolean isError() ***REMOVED***
***REMOVED*** return this.executionException!=null;
***REMOVED*** ***REMOVED***
private String getSubjectHeaderRelease() ***REMOVED*** ***REMOVED*** public void setExecutionException(Exception executionException) ***REMOVED***
return "[DataMinerGhostProductionInstallationRequestReport]"; ***REMOVED*** this.executionException = executionException;
***REMOVED*** ***REMOVED*** ***REMOVED***
private boolean isError() ***REMOVED*** public abstract String getSuccessSubject();
return this.executionException!=null;
***REMOVED***
public void setExecutionException(Exception executionException) ***REMOVED***
this.executionException = executionException;
***REMOVED***
public String getSuccessSubject() ***REMOVED***
return this.getSubjectHeader()+" is SUCCESS";
***REMOVED***
public String getSuccessSubjectRelease() ***REMOVED*** public abstract String getFailedSubject();
return this.getSubjectHeaderRelease()+" is SUCCESS";
***REMOVED***
public String getFailedSubjectRelease() ***REMOVED***
return this.getSubjectHeaderRelease()+" is FAILED";
***REMOVED***
public String getFailedSubject() ***REMOVED***
return String.format(this.getSubjectHeader()+" is FAILED");
***REMOVED***
public String getSuccessBody(String info) ***REMOVED*** public String getSuccessBody(String info) ***REMOVED***
String message = String.format("The installation of the algorithm is completed successfully."); String message = String.format("The installation of the algorithm is completed successfully.");
message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info; message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info;

38
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java Executable file
View File

@ -0,0 +1,38 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
***REMOVED***
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
public class SVNRepositoryManager ***REMOVED***
private SVNRepository svnRepository;
private static SVNRepositoryManager instance;
private SVNRepositoryManager (Configuration configuration) throws SVNException
***REMOVED***
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
this.svnRepository.setAuthenticationManager(authManager);
***REMOVED***
public static SVNRepositoryManager getInstance (Configuration configuration) throws SVNException
***REMOVED***
if (instance == null) instance = new SVNRepositoryManager(configuration);
return instance;
***REMOVED***
public SVNRepository getSvnRepository() ***REMOVED***
return svnRepository;
***REMOVED***
***REMOVED***

153
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
View File

@ -21,33 +21,28 @@ import java.util.TimeZone;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
***REMOVED*** ***REMOVED***
import org.tmatesoft.svn.core.SVNNodeKind; import org.tmatesoft.svn.core.SVNNodeKind;
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream; import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
import org.tmatesoft.svn.core.io.ISVNEditor; import org.tmatesoft.svn.core.io.ISVNEditor;
import org.tmatesoft.svn.core.io.SVNRepository; import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator; import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
/** /**
* Created by ggiammat on 5/9/17. * Created by ggiammat on 5/9/17.
*/ */
public class SVNUpdater ***REMOVED*** public abstract class SVNUpdater ***REMOVED***
private SVNRepository svnRepository; private SVNRepository svnRepository;
private Configuration configuration;
public SVNUpdater(Configuration configuration) throws SVNException ***REMOVED***
public SVNUpdater() throws SVNException ***REMOVED*** this.configuration = configuration;
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getSVNRepository())); this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository();
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
this.svnRepository.setAuthenticationManager(authManager);
***REMOVED*** ***REMOVED***
***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) ***REMOVED*** ***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) ***REMOVED***
@ -58,113 +53,14 @@ public class SVNUpdater ***REMOVED***
public String getDependencyFile(String language/*, String env*/)***REMOVED*** public String getDependencyFile(String language/*, String env*/)
***REMOVED***
String a = ""; return getDependencyFile(this.configuration,language);
***REMOVED*** if (env.equals("Dev"))***REMOVED***
***REMOVED*** a= this.getDevDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Prod"))***REMOVED***
***REMOVED*** a= this.getProdDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Proto"))***REMOVED***
***REMOVED*** a= this.getRProtoDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Preprod"))***REMOVED***
***REMOVED*** a= this.getPreProdDependencyFile(language);
***REMOVED*** ***REMOVED***
a = this.getStagingDependencyFile(language);
return a;
***REMOVED*** ***REMOVED***
public String getDependencyFileProd(String language/*, String env*/)***REMOVED***
String a = "";
***REMOVED*** if (env.equals("Dev"))***REMOVED***
***REMOVED*** a= this.getDevDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Prod"))***REMOVED***
***REMOVED*** a= this.getProdDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Proto"))***REMOVED***
***REMOVED*** a= this.getRProtoDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Preprod"))***REMOVED***
***REMOVED*** a= this.getPreProdDependencyFile(language);
***REMOVED*** ***REMOVED***
a = this.getProdDependencyFile(language);
return a;
***REMOVED***
public String getStagingDependencyFile(String language) ***REMOVED***
return getDependencyFile(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(),language);
***REMOVED*** switch (language) ***REMOVED***
***REMOVED*** case "R":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingCRANDepsList();
***REMOVED*** case "R-blackbox":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingRBDepsList();
***REMOVED*** case "Java":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingJavaDepsList();
***REMOVED*** case "Knime-Workflow":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingKWDepsList();
***REMOVED*** case "Linux-compiled":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingLinuxCompiledDepsList();
***REMOVED*** case "Octave":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingOctaveDepsList();
***REMOVED*** case "Python":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingPythonDepsList();
***REMOVED*** case "Pre-Installed":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingPreInstalledDepsList();
***REMOVED*** case "Windows-compiled":
***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingWCDepsList();
***REMOVED*** default:
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
public String getProdDependencyFile(String language) ***REMOVED***
return getDependencyFile(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(),language);
***REMOVED*** switch (language) ***REMOVED***
***REMOVED*** case "R":
***REMOVED*** return this.configuration.getSVNProdCRANDepsList();
***REMOVED*** case "R-blackbox":
***REMOVED*** return this.configuration.getSVNProdRBDepsList();
***REMOVED*** case "Java":
***REMOVED*** return this.configuration.getSVNProdJavaDepsList();
***REMOVED*** case "Knime-Workflow":
***REMOVED*** return this.configuration.getSVNProdKWDepsList();
***REMOVED*** case "Linux-compiled":
***REMOVED*** return this.configuration.getSVNProdLinuxCompiledDepsList();
***REMOVED*** case "Octave":
***REMOVED*** return this.configuration.getSVNProdOctaveDepsList();
***REMOVED*** case "Python":
***REMOVED*** return this.configuration.getSVNProdPythonDepsList();
***REMOVED*** case "Pre-Installed":
***REMOVED*** return this.configuration.getSVNProdPreInstalledDepsList();
***REMOVED*** case "Windows-compiled":
***REMOVED*** return this.configuration.getSVNProdWCDepsList();
***REMOVED*** default:
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
private String getDependencyFile (Configuration configuration, String language) private String getDependencyFile (Configuration configuration, String language)
***REMOVED*** ***REMOVED***
@ -193,29 +89,24 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED**
***REMOVED*** ***REMOVED***
public void updateSVNStagingAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) ***REMOVED*** public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/)
this.updateSVNAlgorithmList(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); ***REMOVED***
***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) ***REMOVED***
this.updateSVNAlgorithmList(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
***REMOVED*** ***REMOVED***
public void updateAlgorithmFiles(File a) throws SVNException***REMOVED*** public void updateAlgorithmFiles(File a) throws SVNException***REMOVED***
***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a);
this.updateAlgorithmList(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getRepository(), a); this.updateAlgorithmList(this.configuration.getRepository(), a);
***REMOVED*** ***REMOVED***
public void updateAlgorithmFilesProd(File a) throws SVNException***REMOVED***
***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a);
this.updateAlgorithmList(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getRepository(), a);
***REMOVED***
public void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException ***REMOVED*** private void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException ***REMOVED***
try ***REMOVED*** try ***REMOVED***
System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo); System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo);
@ -312,7 +203,7 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED**
***REMOVED*** ***REMOVED***
public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) ***REMOVED*** private void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) ***REMOVED***
try ***REMOVED*** try ***REMOVED***
System.out.println("Updating algorithm list: " + file); System.out.println("Updating algorithm list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
@ -404,10 +295,10 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED**
System.out.println("Checking dependencies list: " + file); System.out.println("Checking dependencies list: " + file);
CheckMethod cm = new CheckMethod();
List<String> validDependencies = new LinkedList<String>(); List<String> validDependencies = new LinkedList<String>();
for (String singlefile: cm.getFiles(file))***REMOVED*** for (String singlefile: CheckMethod.getFiles(file))***REMOVED***
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(singlefile, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); svnRepository.getFile(singlefile, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
@ -477,8 +368,8 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED**
***REMOVED*** ***REMOVED***
public boolean checkIfAvaialable(String file, Collection<Dependency> deps) throws SVNException ***REMOVED*** public boolean checkIfAvaialable(String file, Collection<Dependency> deps) throws SVNException ***REMOVED***
SendMail sm = new SendMail(); ***REMOVED***SendMail sm = new SendMail();
NotificationHelper nh = new NotificationHelper(); ***REMOVED***NotificationHelper nh = new NotificationHelper();
boolean check = false; boolean check = false;
try ***REMOVED*** try ***REMOVED***
@ -608,7 +499,7 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED**
SVNUpdater c = new SVNUpdater(); ***REMOVED*** SVNUpdater c = new SVNUpdater();
***REMOVED***File a = new File("/home/ngalante/Desktop/testCiro"); ***REMOVED***File a = new File("/home/ngalante/Desktop/testCiro");
***REMOVED***File b = new File ("/home/ngalante/Desktop/testB"); ***REMOVED***File b = new File ("/home/ngalante/Desktop/testB");

36
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java
View File

@ -8,6 +8,7 @@ import java.io.BufferedReader;
***REMOVED*** ***REMOVED***
import java.io.InputStreamReader; import java.io.InputStreamReader;
import java.io.OutputStreamWriter; import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
import java.net.URLConnection; import java.net.URLConnection;
@ -23,16 +24,10 @@ import org.apache.http.util.EntityUtils;
import org.gcube.common.authorization.library.AuthorizationEntry; import org.gcube.common.authorization.library.AuthorizationEntry;
***REMOVED*** ***REMOVED***
import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.ServiceEndpoint;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.gcube.smartgears.ContextProvider;
import org.gcube.smartgears.context.application.ApplicationContext;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
@ -43,12 +38,14 @@ public class SendMail ***REMOVED***
***REMOVED*** TODO Auto-generated constructor stub ***REMOVED*** TODO Auto-generated constructor stub
***REMOVED*** ***REMOVED***
public void sendNotification(String subject, String body) throws Exception ***REMOVED*** public void sendNotification(String subject, String body) throws EMailException ***REMOVED***
***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get()); ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get());
***REMOVED***String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); ***REMOVED***String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get());
String serviceAddress = this.getSocialService(); String serviceAddress = this.getSocialService();
@ -62,18 +59,25 @@ public class SendMail ***REMOVED***
***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage); ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage);
***REMOVED*** put the sender, the recipients, subject and body of the mail here ***REMOVED*** put the sender, the recipients, subject and body of the mail here
subject = URLEncoder.encode(subject, "UTF-8");
body = URLEncoder.encode(body, "UTF-8"); try
***REMOVED***
subject = URLEncoder.encode(subject, "UTF-8");
body = URLEncoder.encode(body, "UTF-8");
***REMOVED***
catch (UnsupportedEncodingException e)
***REMOVED***
throw new EMailException(e);
***REMOVED***
String requestParameters = "&sender=dataminer&recipients=" + this.getAdmins() + "&subject=" + subject + "&body=" String requestParameters = "&sender=dataminer&recipients=" + this.getAdmins() + "&subject=" + subject + "&body="
+ body; + body;
String response = this.sendPostRequest(requestForMessage, requestParameters); String response = this.sendPostRequest(requestForMessage, requestParameters);
***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK "); ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK ");
if (response == null) ***REMOVED*** if (response == null) throw new EMailException();
Exception e = new Exception("Error in email sending response");
throw e;
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public void notifySubmitter(String a, String b) throws Exception ***REMOVED*** ***REMOVED*** public void notifySubmitter(String a, String b) throws Exception ***REMOVED***
@ -191,7 +195,7 @@ public class SendMail ***REMOVED***
public String getAdmins() throws Exception ***REMOVED*** public String getAdmins()***REMOVED***
try***REMOVED*** try***REMOVED***
List<String> s = new LinkedList<String>(); List<String> s = new LinkedList<String>();
JSONObject obj = new JSONObject(this.getAdminRoles()); JSONObject obj = new JSONObject(this.getAdminRoles());
@ -206,7 +210,7 @@ public class SendMail ***REMOVED***
***REMOVED*** ***REMOVED***
return s.toString().replace("[", "").replace("]", ""); return s.toString().replace("[", "").replace("]", "");
***REMOVED*** ***REMOVED***
catch(Exception a)***REMOVED***return "ngalante, lucio.lelii, roberto.cirillo, gianpaolo.coro, giancarlo.panichi, scarponi"; ***REMOVED*** catch(Exception a)***REMOVED***return "ciro.formisano, lucio.lelii, roberto.cirillo, gianpaolo.coro, giancarlo.panichi, scarponi"; ***REMOVED***
***REMOVED*** ***REMOVED***

19
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java Executable file
View File

@ -0,0 +1,19 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
public class EMailException extends Exception ***REMOVED***
***REMOVED****
*
***REMOVED***
private static final long serialVersionUID = 1L;
public EMailException() ***REMOVED***
super ("Unable to send email notification");
***REMOVED***
public EMailException(Throwable cause) ***REMOVED***
super ("Unable to send email notification",cause);
***REMOVED***
***REMOVED***

272
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java Executable file
View File

@ -0,0 +1,272 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import java.io.File;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
***REMOVED***
***REMOVED***
import com.jcraft.jsch.Channel;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
public class CheckMethodProduction extends CheckMethod***REMOVED***
private Logger logger;
public CheckMethodProduction()
***REMOVED***
this.logger = LoggerFactory.getLogger(CheckMethodProduction.class);
***REMOVED***
***REMOVED***
public boolean algoExists(Algorithm algo/*, String env*/) throws Exception***REMOVED***
this.logger.debug("Looking if algo "+algo.getName()+ " exists in production");
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))***REMOVED***
this.copyFromDmToSVN(file/*,env*/);
this.copyFromDmToSVN(file2/*,env*/);
return true;
***REMOVED***
else
System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
return false;
***REMOVED***
***REMOVED***
public void deleteFiles(Algorithm a/*,String env*/) throws Exception
***REMOVED***
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
System.out.println("checking existing in env: " + productionConfiguration.getHost());
File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/)))***REMOVED***
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
***REMOVED***
else System.out.println("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
***REMOVED***
***REMOVED***
public boolean doesExist(String path/*, String env*/) throws Exception ***REMOVED***
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
boolean success = false;
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
System.out.println(path);
try ***REMOVED***
c.lstat(path);
success = true;
***REMOVED*** catch (SftpException e) ***REMOVED***
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) ***REMOVED***
***REMOVED*** file doesn't exist
success = false;
***REMOVED***
***REMOVED***success = true; ***REMOVED*** something else went wrong
***REMOVED***
channel.disconnect();
c.disconnect();
session.disconnect();
return success;
***REMOVED***
***REMOVED***
public void copyFromDmToSVN(File a/*,String env*/) throws Exception ***REMOVED***
JSch jsch = new JSch();
Session session = null;
SVNUpdater svnUpdater = new SVNUpdaterProduction();
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
ChannelSftp sftp = (ChannelSftp) channel;
sftp.cd(productionConfiguration.getGhostAlgoDirectory());
System.out.println("REMOTE : "+productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
System.out.println("LOCAL : /tmp/"+a.getName());
sftp.get(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
channel.disconnect();
session.disconnect();
File f = new File("/tmp/"+a.getName());
svnUpdater.updateAlgorithmFiles(f);
f.delete();
***REMOVED***
public static void main(String[] args) throws Exception ***REMOVED***
***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
***REMOVED*** System.out.println(a.getStagingHost());
CheckMethodProduction a = new CheckMethodProduction();
***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar");
***REMOVED*** System.out.println(aa.getName());
***REMOVED*** System.out.println(aa.getPath());
***REMOVED***a.copyFromDmToSVN(aa);
***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "***REMOVED***"))***REMOVED***
***REMOVED*** System.out.println("AAA");***REMOVED***
***REMOVED***
***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar"))***REMOVED***
***REMOVED*** System.out.println("BBBB");
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar"))***REMOVED***
***REMOVED*** System.out.println("CCCC");***REMOVED***
***REMOVED***
***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar");
***REMOVED*** a.copyFromDmToSVN(aa, "Dev");
***REMOVED***
***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org",
***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548"));
***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
***REMOVED*** "***REMOVED***"));
System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org",
"***REMOVED***"));
***REMOVED***Algorithm aa = new Algorithm();
***REMOVED***aa.setName("UDPIPE_WRAPPER");
***REMOVED***System.out.println(a.algoExists(aa));
***REMOVED******REMOVED***
***REMOVED***ServiceConfiguration bp = new ServiceConfiguration();
***REMOVED******REMOVED***
***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***");
***REMOVED******REMOVED***
***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); ***REMOVED***
***REMOVED***System.out.println("ciao");
***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***Algorithm al = new Algorithm();
***REMOVED*** al.setName("UDPIPE_WRAPPER");
***REMOVED*** a.deleteFiles(al);
***REMOVED***
***REMOVED***

288
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java Executable file
View File

@ -0,0 +1,288 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import java.io.File;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
***REMOVED***
***REMOVED***
import com.jcraft.jsch.Channel;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
public class CheckMethodStaging extends CheckMethod***REMOVED***
private Logger logger;
public CheckMethodStaging()
***REMOVED***
this.logger = LoggerFactory.getLogger(CheckMethodStaging.class);
***REMOVED***
***REMOVED***
public boolean algoExists(Algorithm algo/*, String env*/) throws Exception***REMOVED***
this.logger.debug("Looking if algo "+algo.getName()+ " exists");
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath());
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))
***REMOVED***
this.logger.debug("Files found");
this.copyFromDmToSVN(file/*,env*/);
this.copyFromDmToSVN(file2/*,env*/);
System.out.println("Files have been copied to SVN");
return true;
***REMOVED***
else
***REMOVED***
this.logger.debug("Files not found");
System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
return false;
***REMOVED***
***REMOVED***
***REMOVED***
public void deleteFiles(Algorithm a/*,String env*/) throws Exception***REMOVED***
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
System.out.println("checking existing in env: " + stagingConfiguration.getHost());
File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", stagingConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/)))***REMOVED***
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
***REMOVED***
else System.out.println("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
***REMOVED***
***REMOVED***
public boolean doesExist(String path/*, String env*/) throws Exception ***REMOVED***
this.logger.debug("Looking if file "+path + " exists");
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
boolean success = false;
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
this.logger.debug("Staging configuration host "+stagingConfiguration.getHost());
session = jsch.getSession("root",stagingConfiguration.getHost() );
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
***REMOVED***SftpATTRS is = null;
System.out.println(path);
try ***REMOVED***
c.lstat(path);
this.logger.debug("File found");
success = true;
***REMOVED*** catch (SftpException e)
***REMOVED***
this.logger.error("File not found",e);
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE)
***REMOVED***
***REMOVED*** file doesn't exist
success = false;
***REMOVED***
***REMOVED***success = true; ***REMOVED*** something else went wrong
***REMOVED***
channel.disconnect();
c.disconnect();
session.disconnect();
this.logger.debug("Operation result "+success);
return success;
***REMOVED***
***REMOVED***
public void copyFromDmToSVN(File a/*,String env*/) throws Exception ***REMOVED***
JSch jsch = new JSch();
Session session = null;
SVNUpdater svnUpdater = new SVNUpdaterStaging();
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", stagingConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
ChannelSftp sftp = (ChannelSftp) channel;
sftp.cd(stagingConfiguration.getGhostAlgoDirectory());
System.out.println("REMOTE : "+stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
System.out.println("LOCAL : /tmp/"+a.getName());
sftp.get(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
channel.disconnect();
session.disconnect();
File f = new File("/tmp/"+a.getName());
svnUpdater.updateAlgorithmFiles(f);
f.delete();
***REMOVED***
public static void main(String[] args) throws Exception ***REMOVED***
***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
***REMOVED*** System.out.println(a.getStagingHost());
CheckMethodStaging a = new CheckMethodStaging();
***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar");
***REMOVED*** System.out.println(aa.getName());
***REMOVED*** System.out.println(aa.getPath());
***REMOVED***a.copyFromDmToSVN(aa);
***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "***REMOVED***"))***REMOVED***
***REMOVED*** System.out.println("AAA");***REMOVED***
***REMOVED***
***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar"))***REMOVED***
***REMOVED*** System.out.println("BBBB");
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar"))***REMOVED***
***REMOVED*** System.out.println("CCCC");***REMOVED***
***REMOVED***
***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar");
***REMOVED*** a.copyFromDmToSVN(aa, "Dev");
***REMOVED***
***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org",
***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548"));
***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
***REMOVED*** "***REMOVED***"));
System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org",
"***REMOVED***"));
***REMOVED***Algorithm aa = new Algorithm();
***REMOVED***aa.setName("UDPIPE_WRAPPER");
***REMOVED***System.out.println(a.algoExists(aa));
***REMOVED******REMOVED***
***REMOVED***ServiceConfiguration bp = new ServiceConfiguration();
***REMOVED******REMOVED***
***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***");
***REMOVED******REMOVED***
***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); ***REMOVED***
***REMOVED***System.out.println("ciao");
***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***Algorithm al = new Algorithm();
***REMOVED*** al.setName("UDPIPE_WRAPPER");
***REMOVED*** a.deleteFiles(al);
***REMOVED***
***REMOVED***

14
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java Executable file
View File

@ -0,0 +1,14 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
public class ClusterBuilderProduction extends ClusterBuilder***REMOVED***
public ClusterBuilderProduction() ***REMOVED***
super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration());
***REMOVED***
***REMOVED***

13
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java Executable file
View File

@ -0,0 +1,13 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
public class ClusterBuilderStaging extends ClusterBuilder***REMOVED***
public ClusterBuilderStaging() ***REMOVED***
super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration());
***REMOVED***
***REMOVED***

30
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java Executable file
View File

@ -0,0 +1,30 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
***REMOVED***import scala.actors.threadpool.Arrays;
public class NotificationHelperProduction extends NotificationHelper***REMOVED***
***REMOVED*** private Exception executionException;
private String getSubjectHeader() ***REMOVED***
return "[DataMinerGhostProductionInstallationRequestReport]";
***REMOVED***
@Override
public String getSuccessSubject() ***REMOVED***
return this.getSubjectHeader()+" is SUCCESS";
***REMOVED***
@Override
public String getFailedSubject() ***REMOVED***
return String.format(this.getSubjectHeader()+" is FAILED");
***REMOVED***
***REMOVED***

32
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java Executable file
View File

@ -0,0 +1,32 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
***REMOVED***import scala.actors.threadpool.Arrays;
public class NotificationHelperStaging extends NotificationHelper ***REMOVED***
***REMOVED*** private Exception executionException;
private String getSubjectHeader() ***REMOVED***
return "[DataMinerGhostStagingInstallationRequestReport]";
***REMOVED***
@Override
public String getSuccessSubject() ***REMOVED***
return this.getSubjectHeader()+" is SUCCESS";
***REMOVED***
@Override
public String getFailedSubject() ***REMOVED***
return String.format(this.getSubjectHeader()+" is FAILED");
***REMOVED***
***REMOVED***

21
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java Executable file
View File

@ -0,0 +1,21 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
***REMOVED***
/**
* Created by ggiammat on 5/9/17.
*/
public class SVNUpdaterProduction extends SVNUpdater***REMOVED***
public SVNUpdaterProduction() throws SVNException ***REMOVED***
super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration());
***REMOVED***
***REMOVED***

21
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java Executable file
View File

@ -0,0 +1,21 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
***REMOVED***
/**
* Created by ggiammat on 5/9/17.
*/
public class SVNUpdaterStaging extends SVNUpdater***REMOVED***
public SVNUpdaterStaging() throws SVNException ***REMOVED***
super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration());
***REMOVED***
***REMOVED***

3
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java
View File

@ -6,6 +6,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction;
***REMOVED*** ***REMOVED***
public class JobTest ***REMOVED*** public class JobTest ***REMOVED***
@ -14,7 +15,7 @@ public class JobTest ***REMOVED***
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SVNUpdater svnUpdater = new SVNUpdater(); SVNUpdater svnUpdater = new SVNUpdaterProduction();
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0"); Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0");
***REMOVED***test phase ***REMOVED***test phase

5
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java
View File

@ -5,6 +5,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging;
***REMOVED*** ***REMOVED***
/** /**
@ -17,7 +18,7 @@ public class SVNTests ***REMOVED***
public static void main(String[] args) throws SVNException, IOException, InterruptedException ***REMOVED*** public static void main(String[] args) throws SVNException, IOException, InterruptedException ***REMOVED***
SVNUpdater svnUpdater = new SVNUpdater(); SVNUpdater svnUpdater = new SVNUpdaterStaging();
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0");
***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); ***REMOVED***algo.setClazz(algo.getClazz() + "TEST");
@ -27,7 +28,7 @@ public class SVNTests ***REMOVED***
***REMOVED***System.out.println(algo.getCategory()); ***REMOVED***System.out.println(algo.getCategory());
***REMOVED***System.out.println(algo.getSkipJava()); ***REMOVED***System.out.println(algo.getSkipJava());
svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), ""); ***REMOVED***svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), "");
***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto");
***REMOVED***svnUpdater.readRPRotoDeps(algo); ***REMOVED***svnUpdater.readRPRotoDeps(algo);