From 325f3a3b86aff01939f25c4f10d9c68263b3ec4d Mon Sep 17 00:00:00 2001 From: Ciro Formisano Date: Wed, 29 Nov 2017 15:29:01 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@158895 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 2 +- .../poolmanager/ansible/AnsibleWorker.java | 44 +- .../ansiblebridge/AnsibleBridge.java | 17 +- .../poolmanager/service/DMPMJob.java | 219 ++++++++- .../service/DataminerPoolManager.java | 22 +- .../service/ProductionPublishingJob.java | 144 +----- .../poolmanager/service/StagingJob.java | 152 +----- .../poolmanager/util/CheckMethod.java | 436 +----------------- .../poolmanager/util/CheckPermission.java | 6 +- .../poolmanager/util/ClusterBuilder.java | 51 +- .../poolmanager/util/NotificationHelper.java | 51 +- .../util/SVNRepositoryManager.java | 38 ++ .../poolmanager/util/SVNUpdater.java | 153 +----- .../dataminer/poolmanager/util/SendMail.java | 36 +- .../util/exception/EMailException.java | 19 + .../util/impl/CheckMethodProduction.java | 272 +++++++++++ .../util/impl/CheckMethodStaging.java | 288 ++++++++++++ .../util/impl/ClusterBuilderProduction.java | 14 + .../util/impl/ClusterBuilderStaging.java | 13 + .../impl/NotificationHelperProduction.java | 30 ++ .../util/impl/NotificationHelperStaging.java | 32 ++ .../util/impl/SVNUpdaterProduction.java | 21 + .../util/impl/SVNUpdaterStaging.java | 21 + .../dataminerpoolmanager/JobTest.java | 3 +- .../dataminerpoolmanager/SVNTests.java | 5 +- 25 files changed, 1087 insertions(+), 1002 deletions(-) create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java diff --git a/pom.xml b/pom.xml index 1b4f372..a082141 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ org.gcube.dataAnalysis dataminer-pool-manager war - 2.2.0-SNAPSHOT + 2.3.0-SNAPSHOT dataminer-pool-manager diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index 7c05031..3b802ce 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -1,41 +1,16 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.File; ***REMOVED*** import java.io.InputStream; -import java.io.InputStreamReader; import java.io.PrintStream; -import java.io.PrintWriter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -***REMOVED*** import java.util.Scanner; -***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; ***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -***REMOVED*** -import org.tmatesoft.svn.core.SVNURL; -import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; -import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; -import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream; -import org.tmatesoft.svn.core.io.ISVNEditor; -import org.tmatesoft.svn.core.io.SVNRepository; -import org.tmatesoft.svn.core.io.SVNRepositoryFactory; -import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator; -import org.tmatesoft.svn.core.wc.SVNWCUtil; /** * This class is responsible for the interface with ansible, retrieving log, @@ -72,12 +47,12 @@ public class AnsibleWorker ***REMOVED*** this.ensureWorkStructure(); ***REMOVED*** - public File getWorkdir() ***REMOVED*** - return this.workerRoot; - ***REMOVED*** +***REMOVED*** public File getWorkdir() ***REMOVED*** +***REMOVED*** return this.workerRoot; +***REMOVED*** ***REMOVED*** public File getRolesDir() ***REMOVED*** - return new File(this.getWorkdir(), ROLES_DIR); + return new File(this.workerRoot, ROLES_DIR); ***REMOVED*** public String getWorkerId() ***REMOVED*** @@ -86,20 +61,20 @@ public class AnsibleWorker ***REMOVED*** public void ensureWorkStructure() ***REMOVED*** ***REMOVED*** generate root - this.getWorkdir().mkdirs(); + this.workerRoot.mkdirs(); ***REMOVED*** public void removeWorkStructure() ***REMOVED*** ***REMOVED*** remove the working dir - this.getWorkdir().delete(); + this.workerRoot.delete(); ***REMOVED*** public File getPlaybookFile() ***REMOVED*** - return new File(this.getWorkdir(), PLAYBOOK_NAME); + return new File(this.workerRoot, PLAYBOOK_NAME); ***REMOVED*** public File getInventoryFile() ***REMOVED*** - return new File(this.getWorkdir(), INVENTORY_NAME); + return new File(this.workerRoot, INVENTORY_NAME); ***REMOVED*** @@ -122,7 +97,7 @@ public class AnsibleWorker ***REMOVED*** public int execute(PrintStream ps) throws IOException, InterruptedException, SVNException ***REMOVED*** - System.out.println(this.getWorkdir()); + System.out.println(this.workerRoot); try ***REMOVED*** Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath()); @@ -149,6 +124,7 @@ public class AnsibleWorker ***REMOVED*** while (sc.hasNextLine()) ***REMOVED*** dest.println(sc.nextLine()); ***REMOVED*** + sc.close(); ***REMOVED*** ***REMOVED***).start(); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index 7741953..19a6e41 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -1,10 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; import java.io.File; -import java.io.FileOutputStream; ***REMOVED*** -import java.io.InputStream; -import java.io.PrintStream; import java.util.Collection; ***REMOVED*** import java.util.Map; @@ -32,21 +29,19 @@ import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; -***REMOVED*** -***REMOVED*** public class AnsibleBridge ***REMOVED*** - private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); + ***REMOVED***private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); private String dpmRoot; - public AnsibleBridge() ***REMOVED*** - this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); - ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager")); - - ***REMOVED*** +***REMOVED*** public AnsibleBridge() ***REMOVED*** +***REMOVED*** this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); +***REMOVED*** ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager")); +***REMOVED*** +***REMOVED*** ***REMOVED*** public AnsibleBridge(String root) ***REMOVED*** this.dpmRoot = root; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java index 9b5b6c7..50d9703 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java @@ -5,34 +5,71 @@ import java.io.FileOutputStream; ***REMOVED*** import java.io.PrintStream; import java.io.PrintWriter; -import java.util.LinkedList; -***REMOVED*** +import java.util.Collection; import java.util.UUID; +***REMOVED*** +***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; +import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; +import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; +import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException; +***REMOVED*** +***REMOVED*** ***REMOVED*** public abstract class DMPMJob ***REMOVED*** + private Configuration configuration; + private String dmpmHomeDirectory; + private SVNUpdater svnUpdater; + private File jobLogs; + private String id; + private Algorithm algorithm; + private Cluster cluster; + private String vREName; + private String category; + private String algorithm_type; + private Logger logger; - protected SVNUpdater svnUpdater; - protected File jobLogs; + private enum STATUS + ***REMOVED*** + PROGRESS ("IN PROGRESS"), + COMPLETED ("COMPLETED"), + FAILED ("FAILED"); + + private String status; + + STATUS (String status) + ***REMOVED*** + this.status = status; + ***REMOVED*** +***REMOVED*** - protected String id; + public DMPMJob(SVNUpdater svnUpdater,Configuration configuration,Algorithm algorithm, Cluster cluster,String vREName, + String category, String algorithm_type)***REMOVED*** + this.logger = LoggerFactory.getLogger(DMPMJob.class); - public DMPMJob(SVNUpdater svnUpdater)***REMOVED*** + this.configuration = configuration; + this.algorithm = algorithm; + this.cluster = cluster; + this.vREName = vREName; + this.category = category; + this.algorithm_type = algorithm_type; + this.svnUpdater = svnUpdater; + this.dmpmHomeDirectory = new String (System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); this.id = UUID.randomUUID().toString(); ***REMOVED***TODO: dmpm work directory should be loaded from configuration file - this.jobLogs = new File(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"+File.separator+"jobs"); - + this.jobLogs = new File(this.dmpmHomeDirectory+File.separator+"jobs"); this.jobLogs.mkdirs(); ***REMOVED*** @@ -56,7 +93,7 @@ public abstract class DMPMJob ***REMOVED*** Cluster dataminerCluster, boolean includeAlgorithmDependencies, String user)***REMOVED*** - AnsibleBridge ansibleBridge = new AnsibleBridge(); + AnsibleBridge ansibleBridge = new AnsibleBridge(this.dmpmHomeDirectory); try ***REMOVED*** return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); ***REMOVED*** catch (IOException e) ***REMOVED*** @@ -66,14 +103,148 @@ public abstract class DMPMJob ***REMOVED*** ***REMOVED*** - protected abstract void execute(); - - protected int executeAnsibleWorker(AnsibleWorker worker) throws IOException, InterruptedException, SVNException***REMOVED*** - File path = new File(worker.getWorkdir() + File.separator + "jobs"); - path.mkdirs(); + public void setStatusInformation(STATUS exitStatus) ***REMOVED*** - File n = new File(this.jobLogs + File.separator + this.id); - FileOutputStream fos = new FileOutputStream(n, true); + + try + ***REMOVED*** + File statusFile = new File (this.jobLogs,this.id + "_exitStatus"); + ***REMOVED***File m = new File ( this.jobLogs + File.separator + this.id + "_exitStatus"); + PrintWriter writer = new PrintWriter(statusFile, "UTF-8"); + writer.println(exitStatus.status); + writer.close(); + ***REMOVED*** catch (Exception e) + ***REMOVED*** + this.logger.error ("Unable to update exit status file with status "+exitStatus.status,e); + ***REMOVED*** + + + +***REMOVED*** + + private void updateLogFile (File logFile, String message) + ***REMOVED*** + try + ***REMOVED*** + PrintWriter writer = new PrintWriter(logFile,"UTF-8"); + writer.print(message); + writer.close(); + ***REMOVED*** catch (Exception e) + ***REMOVED*** + this.logger.error("Unable to log the error message: "+message,e); + ***REMOVED*** + + +***REMOVED*** + + protected abstract void execute (); + + private boolean preInstallation (SendMail sm,NotificationHelper nh, File logFile ) throws SVNException, EMailException + ***REMOVED*** + + this.logger.debug("Checking dependencies..."); + Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies( + this.svnUpdater.getDependencyFile(this.algorithm.getLanguage()), + this.algorithm.getDependencies()); + + if (!undefinedDependencies.isEmpty()) + ***REMOVED*** + this.logger.debug("Some dependencies are not defined"); + String message = "Following dependencies are not defined:\n"; + for (String n : undefinedDependencies) ***REMOVED*** + message += "\n" + n +"\n"; + ***REMOVED*** + this.setStatusInformation(STATUS.FAILED); + String errorMessage = nh.getFailedBody(message+"\n\n"+this.buildInfo()); + this.updateLogFile(logFile, errorMessage); + sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", errorMessage); + return false; + ***REMOVED*** + else return true; +***REMOVED*** + + private void installation (SendMail sm,NotificationHelper nh,CheckMethod methodChecker,File logFile ) throws Exception + ***REMOVED*** + methodChecker.deleteFiles(this.algorithm/*, env*/); + int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.cluster, false, "root"),logFile); + System.out.println("Return code= "+ret); + + if (ret != 0) + ***REMOVED*** + this.logger.debug("Ansible work failed, return code "+ret); + this.setStatusInformation(STATUS.FAILED); + String errorMessage = nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo(); + sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",errorMessage); + ***REMOVED*** + + else if (ret == 0) + ***REMOVED*** + this.logger.debug("Operation completed"); + this.setStatusInformation(STATUS.PROGRESS); + this.logger.debug("Checking the method..."); + + if (methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get())&&(methodChecker.algoExists(this.algorithm))) + ***REMOVED*** + this.logger.debug("Method OK and algo exists"); + System.out.println("Interface check ok!"); + System.out.println("Both the files exist at the correct path!"); + + this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.vREName,this.category, this.algorithm_type, + this.algorithm.getFullname()); + this.setStatusInformation(STATUS.COMPLETED); + sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo())); + return; + ***REMOVED*** else + ***REMOVED*** + this.logger.debug("Operation failed"); + this.setStatusInformation(STATUS.FAILED); + sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm", + nh.getFailedBody( + "\n"+ + "Installation completed but DataMiner Interface not working correctly or files " + + this.algorithm.getName() + ".jar and " + this.algorithm.getName() + + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo()); + + ***REMOVED*** + ***REMOVED*** +***REMOVED*** + + protected void execute(NotificationHelper nh, CheckMethod methodChecker) + ***REMOVED*** + SendMail sm = new SendMail(); + try ***REMOVED*** + + this.logger.debug("Pre installation operations"); + File logFile = new File(this.jobLogs,this.id); + ***REMOVED***File logFile = new File(this.jobLogs + File.separator + this.id); + boolean preInstallationResponse = preInstallation(sm, nh, logFile); + this.logger.debug("Pre installation operation completed with result "+preInstallationResponse); + if (preInstallationResponse) + ***REMOVED*** + this.logger.debug("Installation..."); + installation(sm, nh, methodChecker, logFile); + this.logger.debug("Installation completed"); + ***REMOVED*** + + + + + + ***REMOVED*** catch (EMailException eme) + ***REMOVED*** + this.logger.error("Operation failed and unable to send notification email",eme); + ***REMOVED*** + + catch (Exception e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** + +***REMOVED*** + + + + protected int executeAnsibleWorker(AnsibleWorker worker, File logFile) throws IOException, InterruptedException, SVNException***REMOVED*** + FileOutputStream fos = new FileOutputStream(logFile, true); PrintStream ps = new PrintStream(fos); ***REMOVED*** File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); @@ -81,5 +252,19 @@ public abstract class DMPMJob ***REMOVED*** return worker.execute(ps); ***REMOVED*** + + public String buildInfo() ***REMOVED*** + + return + "\n"+ + "Algorithm details:\n"+"\n"+ + "User: "+this.algorithm.getFullname()+"\n"+ + "Algorithm name: "+this.algorithm.getName()+"\n"+ + "Staging DataMiner Host: "+ this.configuration.getHost()+"\n"+ + "Caller VRE: "+ScopeProvider.instance.get()+"\n"+ + "Target VRE: "+this.vREName+"\n"; +***REMOVED*** + + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 20e499f..65c94cb 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -8,7 +8,10 @@ import java.util.Scanner; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; -import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderProduction; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.ClusterBuilderStaging; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging; ***REMOVED*** ***REMOVED*** ***REMOVED*** @@ -17,14 +20,15 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; public class DataminerPoolManager ***REMOVED*** private Logger logger; - - private SVNUpdater svnUpdater; + private SVNUpdaterStaging svnUpdaterStaging; + private SVNUpdaterProduction svnUpdaterProduction; public DataminerPoolManager() ***REMOVED*** this.logger = LoggerFactory.getLogger(this.getClass()); try ***REMOVED*** ***REMOVED***TODO: read this from configuration - this.svnUpdater = new SVNUpdater(); + this.svnUpdaterStaging = new SVNUpdaterStaging(); + this.svnUpdaterProduction = new SVNUpdaterProduction(); ***REMOVED*** catch (SVNException e) ***REMOVED*** e.printStackTrace(); ***REMOVED*** @@ -37,9 +41,10 @@ public class DataminerPoolManager ***REMOVED*** this.logger.debug("Algo "+algo); this.logger.debug("Category "+category); this.logger.debug("Algo type "+algorithm_type); - Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); + ClusterBuilder stagingClusterBuilder = new ClusterBuilderStaging(); + Cluster stagingCluster = stagingClusterBuilder.getDataminerCluster(); ***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); - DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/); + DMPMJob job = new StagingJob(this.svnUpdaterStaging, algo, stagingCluster, /*rProtoCluster,*/ targetVRE, category, algorithm_type/*,env*/); String id = job.start(); return id; ***REMOVED*** @@ -50,8 +55,9 @@ public class DataminerPoolManager ***REMOVED*** this.logger.debug("Algo "+algo); this.logger.debug("Category "+category); this.logger.debug("Algo type "+algorithm_type); - Cluster prodCluster = ClusterBuilder.getProdDataminerCluster(); - DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/); + ClusterBuilder productionClusterBuilder = new ClusterBuilderProduction(); + Cluster prodCluster = productionClusterBuilder.getDataminerCluster(); + DMPMJob job = new ProductionPublishingJob(this.svnUpdaterProduction, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/); String id = job.start(); return id; ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java index bb624f8..515ec43 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -1,164 +1,44 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; -import java.io.File; import java.io.FileNotFoundException; -import java.io.PrintWriter; import java.io.UnsupportedEncodingException; -import java.util.Collection; -***REMOVED*** -***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; -import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; -import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodProduction; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperProduction; +***REMOVED*** +***REMOVED*** public class ProductionPublishingJob extends DMPMJob ***REMOVED*** - private Algorithm algorithm; - private Cluster prodCluster; - private String targetVREName; - private String category; - private String algorithm_type; + ***REMOVED***private String targetVREToken; ***REMOVED***private String env; - + private Logger logger; public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** - super(svnUpdater); - this.jobLogs = new File( - System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs"); - this.jobLogs.mkdirs(); - this.algorithm = algorithm; - this.prodCluster = prodCluster; - this.targetVREName = targetVREName; - this.category = category; - this.algorithm_type = algorithm_type; - ***REMOVED***this.targetVREToken = targetVREToken; - ***REMOVED***this.env= env; - this.getStatus(0); + super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(),algorithm,prodCluster,targetVREName,category,algorithm_type); + this.logger = LoggerFactory.getLogger(StagingJob.class);***REMOVED*** this.jobLogs = new File( + ***REMOVED*** ***REMOVED*** protected void execute() ***REMOVED*** - SendMail sm = new SendMail(); - NotificationHelper nh = new NotificationHelper(); - CheckMethod b = new CheckMethod(); - try ***REMOVED*** - - Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies( - this.svnUpdater.getDependencyFileProd(this.algorithm.getLanguage()/*,env*/), - this.algorithm.getDependencies()); - - if (!undefinedDependencies.isEmpty()) ***REMOVED*** - - String message = "Following dependencies are not defined:\n"; - for (String n : undefinedDependencies) ***REMOVED*** - message += "\n" + n +"\n"; - ***REMOVED*** - this.getStatus(2); - - sm.sendNotification(nh.getFailedSubjectRelease() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo())); - return; - ***REMOVED*** - b.deleteFilesProd(this.algorithm); - - int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.prodCluster, false, "root")); - - if (ret != 0) ***REMOVED*** - this.getStatus(2); - sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo()); - return; - ***REMOVED*** - - - if (ret == 0) ***REMOVED*** - this.getStatus(0); - - if (b.checkMethod(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost(), SecurityTokenProvider.instance.get())&&(b.algoExistsProd(this.algorithm/*, env*/))) ***REMOVED*** - - System.out.println("Interface check ok!"); - System.out.println("Both the files exist at the correct path!"); - - this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName,this.category, this.algorithm_type, - this.algorithm.getFullname()/*, env*/); - - this.getStatus(9); - sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo())); - return; - ***REMOVED*** else - this.getStatus(2); - sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", - nh.getFailedBody( - "\n"+ - "Installation completed but DataMiner Interface not working correctly or files " - + this.algorithm.getName() + ".jar and " + this.algorithm.getName() - + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo()); - return; - - ***REMOVED*** - ***REMOVED*** catch (Exception e) ***REMOVED*** - try ***REMOVED*** - this.getStatus(0); - ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e1.printStackTrace(); - ***REMOVED*** catch (UnsupportedEncodingException e1) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e1.printStackTrace(); - ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** + this.logger.debug("Executing staging job..."); + super.execute(new NotificationHelperProduction(), new CheckMethodProduction()); ***REMOVED*** - - public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** - File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); - PrintWriter writer = new PrintWriter(m, "UTF-8"); - - String response = ""; - - if (exitstatus == 0) ***REMOVED*** - response = "IN PROGRESS"; - writer.println(response); - ***REMOVED***writer.close(); - ***REMOVED*** - - if (exitstatus == 9) ***REMOVED*** - response = "COMPLETED"; - writer.println(response); - ***REMOVED***writer.close(); - ***REMOVED*** - - if (exitstatus == 2) ***REMOVED*** - response = "FAILED"; - writer.println(response); - ***REMOVED***writer.close(); - ***REMOVED*** - writer.close(); - return response; -***REMOVED*** - public String buildInfo() throws UnsupportedEncodingException***REMOVED*** - - return - "\n"+ - "Algorithm details:\n"+"\n"+ - "User: "+this.algorithm.getFullname()+"\n"+ - "Algorithm name: "+this.algorithm.getName()+"\n"+ - "Staging DataMiner Host: "+ DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost()+"\n"+ - "Caller VRE: "+ScopeProvider.instance.get()+"\n"+ - "Target VRE: "+this.targetVREName+"\n"; -***REMOVED*** + diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index bb922aa..ae76189 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -1,184 +1,42 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; -import java.io.File; import java.io.FileNotFoundException; -import java.io.PrintWriter; import java.io.UnsupportedEncodingException; -import java.util.Collection; -***REMOVED*** -***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; -import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; -import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.CheckMethodStaging; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.NotificationHelperStaging; ***REMOVED*** ***REMOVED*** public class StagingJob extends DMPMJob ***REMOVED*** - private Algorithm algorithm; - private Cluster stagingCluster; - ***REMOVED*** private Cluster rProtoCluster; - private String rProtoVREName; - ***REMOVED***private String env; - private String category; - private String algorithm_type; + private Logger logger; public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, /* Cluster rProtoCluster, */ String rProtoVREName, String category, String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** - super(svnUpdater); + super(svnUpdater,DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(),algorithm,stagingCluster,rProtoVREName,category,algorithm_type); this.logger = LoggerFactory.getLogger(StagingJob.class); - - this.jobLogs = new File( - System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs"); - this.jobLogs.mkdirs(); - - this.algorithm = algorithm; - this.stagingCluster = stagingCluster; - ***REMOVED*** this.rProtoCluster = rProtoCluster; - this.rProtoVREName = rProtoVREName; - ***REMOVED***this.env = env; - this.category = category; - this.algorithm_type = algorithm_type; - - - ***REMOVED***File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); - ***REMOVED***PrintWriter writer = new PrintWriter(m, "UTF-8"); - this.getStatus(0); - ***REMOVED***writer.close(); - ***REMOVED*** ***REMOVED*** protected void execute() ***REMOVED*** this.logger.debug("Executing staging job..."); + super.execute(new NotificationHelperStaging(), new CheckMethodStaging()); - CheckMethod methodChecker = new CheckMethod(); - SendMail sm = new SendMail(); - NotificationHelper nh = new NotificationHelper(); - - try ***REMOVED*** - - this.logger.debug("Checking dependencies..."); - Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies( - this.svnUpdater.getDependencyFile(this.algorithm.getLanguage()/*,env*/), - this.algorithm.getDependencies()); - - if (!undefinedDependencies.isEmpty()) - ***REMOVED*** - this.logger.debug("Some dependencies are not defined"); - String message = "Following dependencies are not defined:\n"; - for (String n : undefinedDependencies) ***REMOVED*** - message += "\n" + n +"\n"; - ***REMOVED*** - this.getStatus(2); - - sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo())); - return; - ***REMOVED*** - - ***REMOVED***before the installation to check if the files exist - methodChecker.deleteFiles(this.algorithm/*, env*/); - - int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); - System.out.println("Return code= "+ret); - - if (ret != 0) - ***REMOVED*** - this.logger.debug("Ansible work failed, return code "+ret); - this.getStatus(2); - sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo()); - return; - ***REMOVED*** - - if (ret == 0) - ***REMOVED*** - this.logger.debug("Operation completed"); - this.getStatus(0); - ***REMOVED***System.out.println("1 - Checking existing in env: "+ env); - - ***REMOVED***System.out.println("2 - Checking existing in env: "+ this.env); - this.logger.debug("Checking the method..."); - - - if (methodChecker.checkMethod(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost(), SecurityTokenProvider.instance.get())&&(methodChecker.algoExists(this.algorithm))) - ***REMOVED*** - this.logger.debug("Method OK and algo exists"); - System.out.println("Interface check ok!"); - System.out.println("Both the files exist at the correct path!"); - - this.svnUpdater.updateSVNStagingAlgorithmList(this.algorithm, this.rProtoVREName,this.category, this.algorithm_type, - this.algorithm.getFullname()/*, env*/); - - this.getStatus(9); - sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo())); - return; - ***REMOVED*** else - ***REMOVED*** - this.logger.debug("Operation failed"); - this.getStatus(2); - sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm", - nh.getFailedBody( - "\n"+ - "Installation completed but DataMiner Interface not working correctly or files " - + this.algorithm.getName() + ".jar and " + this.algorithm.getName() - + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo()); - return; - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch (Exception e) ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** ***REMOVED*** - public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** - File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); - PrintWriter writer = new PrintWriter(m, "UTF-8"); - - String response = ""; - - if (exitstatus == 0) ***REMOVED*** - response = "IN PROGRESS"; - writer.println(response); - ***REMOVED***writer.close(); - ***REMOVED*** - - if (exitstatus == 9) ***REMOVED*** - response = "COMPLETED"; - writer.println(response); - ***REMOVED***writer.close(); - ***REMOVED*** - - if (exitstatus == 2) ***REMOVED*** - response = "FAILED"; - writer.println(response); - ***REMOVED***writer.close(); - ***REMOVED*** - writer.close(); - return response; -***REMOVED*** - public String buildInfo() throws UnsupportedEncodingException***REMOVED*** - return - "\n"+ - "Algorithm details:\n"+"\n"+ - "User: "+this.algorithm.getFullname()+"\n"+ - "Algorithm name: "+this.algorithm.getName()+"\n"+ - "Staging DataMiner Host: "+ DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost()+"\n"+ - "Caller VRE: "+ScopeProvider.instance.get()+"\n"+ - "Target VRE: "+rProtoVREName+"\n"; -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java index 579e49b..a586ab8 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java @@ -10,20 +10,11 @@ import java.util.Arrays; import java.util.LinkedList; ***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; ***REMOVED*** ***REMOVED*** ***REMOVED*** -import com.jcraft.jsch.Channel; -import com.jcraft.jsch.ChannelSftp; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.Session; -import com.jcraft.jsch.SftpATTRS; -import com.jcraft.jsch.SftpException; - -public class CheckMethod ***REMOVED*** +public abstract class CheckMethod ***REMOVED*** private Logger logger; @@ -37,14 +28,14 @@ public class CheckMethod ***REMOVED*** this.logger.debug("Checking method for machine "+machine); this.logger.debug("By using tocken "+token); System.out.println("Machine: " + machine); - String getCapabilitesRequest = new String(); - String getCapabilitesResponse = new String(); +***REMOVED*** String getCapabilitesRequest = new String(); +***REMOVED*** String getCapabilitesResponse = new String(); System.out.println(" Token: " + token); String request = "http:***REMOVED***" + machine + "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token; String response = machine + "___" + token + ".xml"; - getCapabilitesRequest = request; - getCapabilitesResponse = response; +***REMOVED*** getCapabilitesRequest = request; +***REMOVED*** getCapabilitesResponse = response; String baseDescriptionRequest = "http:***REMOVED***" + machine + "/wps/WebProcessingService?Request=DescribeProcess&Service=WPS&Version=1.0.0" + "&gcube-token=" + token + "&Identifier="; @@ -111,359 +102,22 @@ public class CheckMethod ***REMOVED*** - public boolean algoExists(Algorithm algo/*, String env*/) throws Exception***REMOVED*** + public abstract boolean algoExists(Algorithm algo/*, String env*/) throws Exception; - this.logger.debug("Looking if algo "+algo.getName()+ " exists"); - Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); - File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar"); - File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar"); - this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath()); - - System.out.println("First file is located to: "+file.getPath()); - System.out.println("Second file is located to: "+file2.getPath()); - - - if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/))) - ***REMOVED*** - this.logger.debug("Files found"); - this.copyFromDmToSVN(file/*,env*/); - this.copyFromDmToSVN(file2/*,env*/); - System.out.println("Files have been copied to SVN"); - - - return true; - ***REMOVED*** - else - ***REMOVED*** - this.logger.debug("Files not found"); - System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); - return false; - ***REMOVED*** - -***REMOVED*** + public abstract void deleteFiles(Algorithm a/*,String env*/) throws Exception; + - public boolean algoExistsProd(Algorithm a/*, String env*/) throws Exception***REMOVED*** - Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); - File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar"); - File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); - - - System.out.println("First file is located to: "+file.getPath()); - System.out.println("Second file is located to: "+file2.getPath()); - - - if ((this.doesExistProd(file.getPath()/*,env*/)) && (this.doesExistProd(file2.getPath()/*,env*/)))***REMOVED*** - this.copyFromDmToSVNProd(file/*,env*/); - this.copyFromDmToSVNProd(file2/*,env*/); - - - return true; - ***REMOVED*** - else - System.out.println("Algorithm "+a.getName()+".jar"+ " and "+a.getName()+"_interface.jar files are not present at the expected path"); - return false; -***REMOVED*** + public abstract boolean doesExist(String path/*, String env*/) throws Exception; + - public void deleteFiles(Algorithm a/*,String env*/) throws Exception***REMOVED*** - JSch jsch = new JSch(); - Session session = null; - Channel channel = null; - ChannelSftp c = null; - Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); - System.out.println("checking existing in env: " + stagingConfiguration.getHost()); - - File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar"); - File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); - - - System.out.println("First file is located to: "+file.getPath()); - System.out.println("Second file is located to: "+file2.getPath()); - - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", stagingConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - channel = session.openChannel("sftp"); - channel.connect(); - System.out.println("shell channel connected...."); - - c = (ChannelSftp) channel; - - if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/)))***REMOVED*** - - c.rm(file.getPath()); - c.rm(file2.getPath()); - System.out.println("Both the files have been deleted"); - ***REMOVED*** - else System.out.println("Files not found"); - channel.disconnect(); - c.disconnect(); - session.disconnect(); - -***REMOVED*** - - - public void deleteFilesProd(Algorithm a/*,String env*/) throws Exception***REMOVED*** - JSch jsch = new JSch(); - Session session = null; - Channel channel = null; - ChannelSftp c = null; - Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); - System.out.println("checking existing in env: " + productionConfiguration.getHost()); - - File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar"); - File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); - - - System.out.println("First file is located to: "+file.getPath()); - System.out.println("Second file is located to: "+file2.getPath()); - - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", productionConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - channel = session.openChannel("sftp"); - channel.connect(); - System.out.println("shell channel connected...."); - - c = (ChannelSftp) channel; - - if(doesExistProd(file.getPath()/*,env*/)&&(doesExistProd(file2.getPath()/*,env*/)))***REMOVED*** - - c.rm(file.getPath()); - c.rm(file2.getPath()); - System.out.println("Both the files have been deleted"); - ***REMOVED*** - else System.out.println("Files not found"); - channel.disconnect(); - c.disconnect(); - session.disconnect(); - -***REMOVED*** - - - - public boolean doesExist(String path/*, String env*/) throws Exception ***REMOVED*** - this.logger.debug("Looking if file "+path + " exists"); - JSch jsch = new JSch(); - Session session = null; - Channel channel = null; - ChannelSftp c = null; - boolean success = false; - Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - this.logger.debug("Staging configuration host "+stagingConfiguration.getHost()); - session = jsch.getSession("root",stagingConfiguration.getHost() ); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - channel = session.openChannel("sftp"); - channel.connect(); - System.out.println("shell channel connected...."); - - c = (ChannelSftp) channel; - - SftpATTRS is = null; - System.out.println(path); - - try ***REMOVED*** - is = c.lstat(path); - this.logger.debug("File found"); - success = true; - ***REMOVED*** catch (SftpException e) - ***REMOVED*** - this.logger.error("File not found",e); - - if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) - ***REMOVED*** - ***REMOVED*** file doesn't exist - success = false; - ***REMOVED*** - ***REMOVED***success = true; ***REMOVED*** something else went wrong - ***REMOVED*** - channel.disconnect(); - c.disconnect(); - session.disconnect(); - this.logger.debug("Operation result "+success); - return success; - -***REMOVED*** - - - - public boolean doesExistProd(String path/*, String env*/) throws Exception ***REMOVED*** - JSch jsch = new JSch(); - Session session = null; - Channel channel = null; - ChannelSftp c = null; - boolean success = false; - Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); - - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", productionConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - channel = session.openChannel("sftp"); - channel.connect(); - System.out.println("shell channel connected...."); - - c = (ChannelSftp) channel; - - SftpATTRS is = null; - System.out.println(path); - - try ***REMOVED*** - is = c.lstat(path); - success = true; - ***REMOVED*** catch (SftpException e) ***REMOVED*** - if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) ***REMOVED*** - ***REMOVED*** file doesn't exist - success = false; - ***REMOVED*** - ***REMOVED***success = true; ***REMOVED*** something else went wrong - ***REMOVED*** - channel.disconnect(); - c.disconnect(); - session.disconnect(); - return success; - -***REMOVED*** - - - public void copyFromDmToSVN(File a/*,String env*/) throws Exception ***REMOVED*** - JSch jsch = new JSch(); - Session session = null; - SVNUpdater svnUpdater = new SVNUpdater(); - Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); - - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", stagingConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - Channel channel = session.openChannel("sftp"); - channel.connect(); - - ChannelSftp sftp = (ChannelSftp) channel; - - sftp.cd(stagingConfiguration.getGhostAlgoDirectory()); - - System.out.println("REMOTE : "+stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()); - System.out.println("LOCAL : /tmp/"+a.getName()); - - sftp.get(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName()); - - channel.disconnect(); - session.disconnect(); - - File f = new File("/tmp/"+a.getName()); - svnUpdater.updateAlgorithmFiles(f); - f.delete(); -***REMOVED*** + public abstract void copyFromDmToSVN(File a/*,String env*/) throws Exception; - - public void copyFromDmToSVNProd(File a/*,String env*/) throws Exception ***REMOVED*** - JSch jsch = new JSch(); - Session session = null; - SVNUpdater svnUpdater = new SVNUpdater(); - Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", productionConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - Channel channel = session.openChannel("sftp"); - channel.connect(); - - ChannelSftp sftp = (ChannelSftp) channel; - - sftp.cd(productionConfiguration.getGhostAlgoDirectory()); - - System.out.println("REMOTE : "+productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()); - System.out.println("LOCAL : /tmp/"+a.getName()); - - sftp.get(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName()); - - channel.disconnect(); - session.disconnect(); - - File f = new File("/tmp/"+a.getName()); - svnUpdater.updateAlgorithmFilesProd(f); - f.delete(); -***REMOVED*** - public List getFiles(String a)***REMOVED*** + public static List getFiles(String a)***REMOVED*** String[] array = a.split(","); ArrayList list = new ArrayList<>(Arrays.asList(array)); @@ -477,71 +131,5 @@ public class CheckMethod ***REMOVED*** ***REMOVED*** - public static void main(String[] args) throws Exception ***REMOVED*** -***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); -***REMOVED*** System.out.println(a.getStagingHost()); - - CheckMethod a = new CheckMethod(); - - ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); - -***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar"); -***REMOVED*** System.out.println(aa.getName()); -***REMOVED*** System.out.println(aa.getPath()); - - - - - - - ***REMOVED***a.copyFromDmToSVN(aa); -***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "***REMOVED***"))***REMOVED*** -***REMOVED*** System.out.println("AAA");***REMOVED*** -***REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar"))***REMOVED*** -***REMOVED*** System.out.println("BBBB"); -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar"))***REMOVED*** -***REMOVED*** System.out.println("CCCC");***REMOVED*** -***REMOVED*** -***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); -***REMOVED*** a.copyFromDmToSVN(aa, "Dev"); - -***REMOVED*** - - -***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", -***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548")); - -***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", - ***REMOVED*** "***REMOVED***")); - -System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org", - "***REMOVED***")); - - -***REMOVED***Algorithm aa = new Algorithm(); -***REMOVED***aa.setName("UDPIPE_WRAPPER"); -***REMOVED***System.out.println(a.algoExists(aa)); -***REMOVED******REMOVED*** -***REMOVED***ServiceConfiguration bp = new ServiceConfiguration(); -***REMOVED******REMOVED*** -***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***"); -***REMOVED******REMOVED*** -***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); ***REMOVED*** -***REMOVED***System.out.println("ciao"); -***REMOVED*** -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED***Algorithm al = new Algorithm(); -***REMOVED*** al.setName("UDPIPE_WRAPPER"); -***REMOVED*** a.deleteFiles(al); - - - - - -***REMOVED*** -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java index 0ac5551..269924e 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java @@ -1,11 +1,9 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; +import static org.gcube.common.authorization.client.Constants.authorizationService; + ***REMOVED*** import org.gcube.common.authorization.library.AuthorizationEntry; -***REMOVED*** -***REMOVED*** - -import static org.gcube.common.authorization.client.Constants.authorizationService; public class CheckPermission ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java index 7467bae..47ba533 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java @@ -6,58 +6,37 @@ import java.io.FileNotFoundException; ***REMOVED*** ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -public class ClusterBuilder ***REMOVED*** +public abstract class ClusterBuilder ***REMOVED*** + private Configuration configuration; + + public ClusterBuilder (Configuration configuration) + ***REMOVED*** + this.configuration = configuration; +***REMOVED*** ***REMOVED***1. to complete - public static Cluster getStagingDataminerCluster() throws FileNotFoundException***REMOVED*** + public Cluster getDataminerCluster() throws FileNotFoundException***REMOVED*** Cluster cluster = new Cluster(); Host h = new Host(); - h.setName(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getHost()); + h.setName(this.configuration.getHost()); cluster.addHost(h); -***REMOVED*** if (env.equals("Dev"))***REMOVED*** -***REMOVED*** h.setName(p.getDevStagingHost()); -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto"))))***REMOVED*** -***REMOVED*** h.setName(p.getProtoProdStagingHost()); -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** - - return cluster; -***REMOVED*** - - - public static Cluster getProdDataminerCluster() throws FileNotFoundException***REMOVED*** - Cluster cluster = new Cluster(); - Host h = new Host(); - h.setName(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getHost()); - cluster.addHost(h); - -***REMOVED*** if (env.equals("Dev"))***REMOVED*** -***REMOVED*** h.setName(p.getDevStagingHost()); -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto"))))***REMOVED*** -***REMOVED*** h.setName(p.getProtoProdStagingHost()); -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** return cluster; ***REMOVED*** + - public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException***REMOVED*** + + public Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException***REMOVED*** Cluster cluster = new Cluster(); for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) ***REMOVED*** cluster.addHost(h); @@ -65,13 +44,13 @@ public class ClusterBuilder ***REMOVED*** return cluster; ***REMOVED*** - public static Cluster getRProtoCluster() throws IOException***REMOVED*** + public Cluster getRProtoCluster() throws IOException***REMOVED*** ***REMOVED***Assumes the service is running in RPrototypingLab String token = SecurityTokenProvider.instance.get(); String targetVRE = ScopeProvider.instance.get(); - return getVRECluster(token, targetVRE); + return this.getVRECluster(token, targetVRE); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java index bfc1be8..bc30827 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -1,53 +1,28 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; -import java.text.SimpleDateFormat; -import java.util.Calendar; - -import org.apache.commons.lang.StringUtils; - ***REMOVED***import scala.actors.threadpool.Arrays; -public class NotificationHelper ***REMOVED*** +public abstract class NotificationHelper ***REMOVED*** - private Exception executionException; +***REMOVED*** private Exception executionException; - public NotificationHelper() ***REMOVED*** - ***REMOVED*** - private String getSubjectHeader() ***REMOVED*** - return "[DataMinerGhostStagingInstallationRequestReport]"; - ***REMOVED*** + + +***REMOVED*** private boolean isError() ***REMOVED*** +***REMOVED*** return this.executionException!=null; +***REMOVED*** ***REMOVED*** - private String getSubjectHeaderRelease() ***REMOVED*** - return "[DataMinerGhostProductionInstallationRequestReport]"; - ***REMOVED*** +***REMOVED*** public void setExecutionException(Exception executionException) ***REMOVED*** +***REMOVED*** this.executionException = executionException; +***REMOVED*** ***REMOVED*** - private boolean isError() ***REMOVED*** - return this.executionException!=null; - ***REMOVED*** - - public void setExecutionException(Exception executionException) ***REMOVED*** - this.executionException = executionException; - ***REMOVED*** + public abstract String getSuccessSubject(); - public String getSuccessSubject() ***REMOVED*** - return this.getSubjectHeader()+" is SUCCESS"; - ***REMOVED*** - public String getSuccessSubjectRelease() ***REMOVED*** - return this.getSubjectHeaderRelease()+" is SUCCESS"; - ***REMOVED*** - - - public String getFailedSubjectRelease() ***REMOVED*** - return this.getSubjectHeaderRelease()+" is FAILED"; - ***REMOVED*** - + public abstract String getFailedSubject(); - public String getFailedSubject() ***REMOVED*** - return String.format(this.getSubjectHeader()+" is FAILED"); - ***REMOVED*** - + public String getSuccessBody(String info) ***REMOVED*** String message = String.format("The installation of the algorithm is completed successfully."); message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java new file mode 100755 index 0000000..d4c731c --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java @@ -0,0 +1,38 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; +***REMOVED*** +import org.tmatesoft.svn.core.SVNURL; +import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; +import org.tmatesoft.svn.core.io.SVNRepository; +import org.tmatesoft.svn.core.io.SVNRepositoryFactory; +import org.tmatesoft.svn.core.wc.SVNWCUtil; + +public class SVNRepositoryManager ***REMOVED*** + + private SVNRepository svnRepository; + private static SVNRepositoryManager instance; + + private SVNRepositoryManager (Configuration configuration) throws SVNException + ***REMOVED*** + this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository())); + ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); + this.svnRepository.setAuthenticationManager(authManager); +***REMOVED*** + + + public static SVNRepositoryManager getInstance (Configuration configuration) throws SVNException + ***REMOVED*** + if (instance == null) instance = new SVNRepositoryManager(configuration); + + return instance; +***REMOVED*** + + + public SVNRepository getSvnRepository() ***REMOVED*** + return svnRepository; +***REMOVED*** + + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index 8defe91..c6ba30c 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -21,33 +21,28 @@ import java.util.TimeZone; import org.apache.commons.io.FileUtils; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; ***REMOVED*** import org.tmatesoft.svn.core.SVNNodeKind; -import org.tmatesoft.svn.core.SVNURL; -import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream; import org.tmatesoft.svn.core.io.ISVNEditor; import org.tmatesoft.svn.core.io.SVNRepository; -import org.tmatesoft.svn.core.io.SVNRepositoryFactory; import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator; -import org.tmatesoft.svn.core.wc.SVNWCUtil; /** * Created by ggiammat on 5/9/17. */ -public class SVNUpdater ***REMOVED*** +public abstract class SVNUpdater ***REMOVED*** private SVNRepository svnRepository; + private Configuration configuration; - - public SVNUpdater() throws SVNException ***REMOVED*** - this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getSVNRepository())); - ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); - this.svnRepository.setAuthenticationManager(authManager); + public SVNUpdater(Configuration configuration) throws SVNException ***REMOVED*** + this.configuration = configuration; + this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository(); + ***REMOVED*** ***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) ***REMOVED*** @@ -58,113 +53,14 @@ public class SVNUpdater ***REMOVED*** - public String getDependencyFile(String language/*, String env*/)***REMOVED*** - - String a = ""; -***REMOVED*** if (env.equals("Dev"))***REMOVED*** -***REMOVED*** a= this.getDevDependencyFile(language); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Prod"))***REMOVED*** -***REMOVED*** a= this.getProdDependencyFile(language); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Proto"))***REMOVED*** -***REMOVED*** a= this.getRProtoDependencyFile(language); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Preprod"))***REMOVED*** -***REMOVED*** a= this.getPreProdDependencyFile(language); -***REMOVED*** ***REMOVED*** - a = this.getStagingDependencyFile(language); - return a; + public String getDependencyFile(String language/*, String env*/) + ***REMOVED*** + return getDependencyFile(this.configuration,language); ***REMOVED*** -public String getDependencyFileProd(String language/*, String env*/)***REMOVED*** - - String a = ""; -***REMOVED*** if (env.equals("Dev"))***REMOVED*** -***REMOVED*** a= this.getDevDependencyFile(language); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Prod"))***REMOVED*** -***REMOVED*** a= this.getProdDependencyFile(language); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Proto"))***REMOVED*** -***REMOVED*** a= this.getRProtoDependencyFile(language); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Preprod"))***REMOVED*** -***REMOVED*** a= this.getPreProdDependencyFile(language); -***REMOVED*** ***REMOVED*** - a = this.getProdDependencyFile(language); - return a; -***REMOVED*** - - - - public String getStagingDependencyFile(String language) ***REMOVED*** - - return getDependencyFile(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(),language); - -***REMOVED*** switch (language) ***REMOVED*** -***REMOVED*** case "R": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingCRANDepsList(); -***REMOVED*** case "R-blackbox": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingRBDepsList(); -***REMOVED*** case "Java": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingJavaDepsList(); -***REMOVED*** case "Knime-Workflow": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingKWDepsList(); -***REMOVED*** case "Linux-compiled": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingLinuxCompiledDepsList(); -***REMOVED*** case "Octave": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingOctaveDepsList(); -***REMOVED*** case "Python": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingPythonDepsList(); -***REMOVED*** case "Pre-Installed": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingPreInstalledDepsList(); -***REMOVED*** case "Windows-compiled": -***REMOVED*** return DMPMClientConfigurator.getInstance().getSVNStagingWCDepsList(); -***REMOVED*** default: -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** - - - public String getProdDependencyFile(String language) ***REMOVED*** - - - return getDependencyFile(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(),language); - -***REMOVED*** switch (language) ***REMOVED*** -***REMOVED*** case "R": -***REMOVED*** return this.configuration.getSVNProdCRANDepsList(); -***REMOVED*** case "R-blackbox": -***REMOVED*** return this.configuration.getSVNProdRBDepsList(); -***REMOVED*** case "Java": -***REMOVED*** return this.configuration.getSVNProdJavaDepsList(); -***REMOVED*** case "Knime-Workflow": -***REMOVED*** return this.configuration.getSVNProdKWDepsList(); -***REMOVED*** case "Linux-compiled": -***REMOVED*** return this.configuration.getSVNProdLinuxCompiledDepsList(); -***REMOVED*** case "Octave": -***REMOVED*** return this.configuration.getSVNProdOctaveDepsList(); -***REMOVED*** case "Python": -***REMOVED*** return this.configuration.getSVNProdPythonDepsList(); -***REMOVED*** case "Pre-Installed": -***REMOVED*** return this.configuration.getSVNProdPreInstalledDepsList(); -***REMOVED*** case "Windows-compiled": -***REMOVED*** return this.configuration.getSVNProdWCDepsList(); -***REMOVED*** default: -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** - private String getDependencyFile (Configuration configuration, String language) ***REMOVED*** @@ -193,29 +89,24 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED** ***REMOVED*** - public void updateSVNStagingAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) ***REMOVED*** - this.updateSVNAlgorithmList(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); -***REMOVED*** - - public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) ***REMOVED*** - this.updateSVNAlgorithmList(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); + public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) + ***REMOVED*** + this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); ***REMOVED*** + public void updateAlgorithmFiles(File a) throws SVNException***REMOVED*** ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); - this.updateAlgorithmList(DMPMClientConfiguratorManager.getInstance().getStagingConfiguration().getRepository(), a); + this.updateAlgorithmList(this.configuration.getRepository(), a); ***REMOVED*** - public void updateAlgorithmFilesProd(File a) throws SVNException***REMOVED*** - ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); - this.updateAlgorithmList(DMPMClientConfiguratorManager.getInstance().getProductionConfiguration().getRepository(), a); -***REMOVED*** + - public void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException ***REMOVED*** + private void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException ***REMOVED*** try ***REMOVED*** System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo); @@ -312,7 +203,7 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED** ***REMOVED*** - public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) ***REMOVED*** + private void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) ***REMOVED*** try ***REMOVED*** System.out.println("Updating algorithm list: " + file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); @@ -404,10 +295,10 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED** System.out.println("Checking dependencies list: " + file); - CheckMethod cm = new CheckMethod(); + List validDependencies = new LinkedList(); - for (String singlefile: cm.getFiles(file))***REMOVED*** + for (String singlefile: CheckMethod.getFiles(file))***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(singlefile, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); @@ -477,8 +368,8 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED** ***REMOVED*** public boolean checkIfAvaialable(String file, Collection deps) throws SVNException ***REMOVED*** - SendMail sm = new SendMail(); - NotificationHelper nh = new NotificationHelper(); + ***REMOVED***SendMail sm = new SendMail(); + ***REMOVED***NotificationHelper nh = new NotificationHelper(); boolean check = false; try ***REMOVED*** @@ -608,7 +499,7 @@ public String getDependencyFileProd(String language/*, String env*/)***REMOVED** - SVNUpdater c = new SVNUpdater(); + ***REMOVED*** SVNUpdater c = new SVNUpdater(); ***REMOVED***File a = new File("/home/ngalante/Desktop/testCiro"); ***REMOVED***File b = new File ("/home/ngalante/Desktop/testB"); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java index 8598c6c..f0f7bb8 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -8,6 +8,7 @@ import java.io.BufferedReader; ***REMOVED*** import java.io.InputStreamReader; import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; ***REMOVED*** ***REMOVED*** import java.net.URLConnection; @@ -23,16 +24,10 @@ import org.apache.http.util.EntityUtils; import org.gcube.common.authorization.library.AuthorizationEntry; ***REMOVED*** import org.gcube.common.resources.gcore.GCoreEndpoint; -import org.gcube.common.resources.gcore.GenericResource; -import org.gcube.common.resources.gcore.ServiceEndpoint; ***REMOVED*** - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; - +import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -import org.gcube.smartgears.ContextProvider; -import org.gcube.smartgears.context.application.ApplicationContext; import org.json.JSONArray; import org.json.JSONObject; @@ -43,12 +38,14 @@ public class SendMail ***REMOVED*** ***REMOVED*** TODO Auto-generated constructor stub ***REMOVED*** - public void sendNotification(String subject, String body) throws Exception ***REMOVED*** + public void sendNotification(String subject, String body) throws EMailException ***REMOVED*** ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get()); ***REMOVED***String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); + + String serviceAddress = this.getSocialService(); @@ -62,18 +59,25 @@ public class SendMail ***REMOVED*** ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage); ***REMOVED*** put the sender, the recipients, subject and body of the mail here - subject = URLEncoder.encode(subject, "UTF-8"); - body = URLEncoder.encode(body, "UTF-8"); + + try + ***REMOVED*** + subject = URLEncoder.encode(subject, "UTF-8"); + body = URLEncoder.encode(body, "UTF-8"); + ***REMOVED*** + catch (UnsupportedEncodingException e) + ***REMOVED*** + throw new EMailException(e); + ***REMOVED*** + String requestParameters = "&sender=dataminer&recipients=" + this.getAdmins() + "&subject=" + subject + "&body=" + body; String response = this.sendPostRequest(requestForMessage, requestParameters); ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK "); - if (response == null) ***REMOVED*** - Exception e = new Exception("Error in email sending response"); - throw e; - ***REMOVED*** + if (response == null) throw new EMailException(); + ***REMOVED*** ***REMOVED*** public void notifySubmitter(String a, String b) throws Exception ***REMOVED*** @@ -191,7 +195,7 @@ public class SendMail ***REMOVED*** - public String getAdmins() throws Exception ***REMOVED*** + public String getAdmins()***REMOVED*** try***REMOVED*** List s = new LinkedList(); JSONObject obj = new JSONObject(this.getAdminRoles()); @@ -206,7 +210,7 @@ public class SendMail ***REMOVED*** ***REMOVED*** return s.toString().replace("[", "").replace("]", ""); ***REMOVED*** - catch(Exception a)***REMOVED***return "ngalante, lucio.lelii, roberto.cirillo, gianpaolo.coro, giancarlo.panichi, scarponi"; ***REMOVED*** + catch(Exception a)***REMOVED***return "ciro.formisano, lucio.lelii, roberto.cirillo, gianpaolo.coro, giancarlo.panichi, scarponi"; ***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java new file mode 100755 index 0000000..1dae6ad --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/EMailException.java @@ -0,0 +1,19 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.exception; + +public class EMailException extends Exception ***REMOVED*** + + + +***REMOVED**** + * +***REMOVED*** + private static final long serialVersionUID = 1L; + + public EMailException() ***REMOVED*** + super ("Unable to send email notification"); +***REMOVED*** + + public EMailException(Throwable cause) ***REMOVED*** + super ("Unable to send email notification",cause); +***REMOVED*** +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java new file mode 100755 index 0000000..8562759 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java @@ -0,0 +1,272 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; + +import java.io.File; + +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +***REMOVED*** +***REMOVED*** + +import com.jcraft.jsch.Channel; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.SftpException; + +public class CheckMethodProduction extends CheckMethod***REMOVED*** + + private Logger logger; + + public CheckMethodProduction() + ***REMOVED*** + this.logger = LoggerFactory.getLogger(CheckMethodProduction.class); +***REMOVED*** + + + + +***REMOVED*** + public boolean algoExists(Algorithm algo/*, String env*/) throws Exception***REMOVED*** + + this.logger.debug("Looking if algo "+algo.getName()+ " exists in production"); + Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); + File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar"); + File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar"); + + + System.out.println("First file is located to: "+file.getPath()); + System.out.println("Second file is located to: "+file2.getPath()); + + + if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))***REMOVED*** + this.copyFromDmToSVN(file/*,env*/); + this.copyFromDmToSVN(file2/*,env*/); + + + return true; + ***REMOVED*** + else + System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); + return false; +***REMOVED*** + +***REMOVED*** + public void deleteFiles(Algorithm a/*,String env*/) throws Exception + ***REMOVED*** + JSch jsch = new JSch(); + Session session = null; + Channel channel = null; + ChannelSftp c = null; + Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); + System.out.println("checking existing in env: " + productionConfiguration.getHost()); + + File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar"); + File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); + + + System.out.println("First file is located to: "+file.getPath()); + System.out.println("Second file is located to: "+file2.getPath()); + + + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + session = jsch.getSession("root", productionConfiguration.getHost()); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + channel = session.openChannel("sftp"); + channel.connect(); + System.out.println("shell channel connected...."); + + c = (ChannelSftp) channel; + + if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/)))***REMOVED*** + + c.rm(file.getPath()); + c.rm(file2.getPath()); + System.out.println("Both the files have been deleted"); + ***REMOVED*** + else System.out.println("Files not found"); + channel.disconnect(); + c.disconnect(); + session.disconnect(); + +***REMOVED*** + + +***REMOVED*** + public boolean doesExist(String path/*, String env*/) throws Exception ***REMOVED*** + JSch jsch = new JSch(); + Session session = null; + Channel channel = null; + ChannelSftp c = null; + boolean success = false; + Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); + + + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + session = jsch.getSession("root", productionConfiguration.getHost()); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + channel = session.openChannel("sftp"); + channel.connect(); + System.out.println("shell channel connected...."); + + c = (ChannelSftp) channel; + + + System.out.println(path); + + try ***REMOVED*** + c.lstat(path); + success = true; + ***REMOVED*** catch (SftpException e) ***REMOVED*** + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) ***REMOVED*** + ***REMOVED*** file doesn't exist + success = false; + ***REMOVED*** + ***REMOVED***success = true; ***REMOVED*** something else went wrong + ***REMOVED*** + channel.disconnect(); + c.disconnect(); + session.disconnect(); + return success; + +***REMOVED*** + + +***REMOVED*** + public void copyFromDmToSVN(File a/*,String env*/) throws Exception ***REMOVED*** + JSch jsch = new JSch(); + Session session = null; + SVNUpdater svnUpdater = new SVNUpdaterProduction(); + Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); + + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + session = jsch.getSession("root", productionConfiguration.getHost()); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + Channel channel = session.openChannel("sftp"); + channel.connect(); + + ChannelSftp sftp = (ChannelSftp) channel; + + sftp.cd(productionConfiguration.getGhostAlgoDirectory()); + + System.out.println("REMOTE : "+productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()); + System.out.println("LOCAL : /tmp/"+a.getName()); + + sftp.get(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName()); + + channel.disconnect(); + session.disconnect(); + + File f = new File("/tmp/"+a.getName()); + svnUpdater.updateAlgorithmFiles(f); + f.delete(); +***REMOVED*** + + + + public static void main(String[] args) throws Exception ***REMOVED*** +***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); +***REMOVED*** System.out.println(a.getStagingHost()); + + CheckMethodProduction a = new CheckMethodProduction(); + + ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); + +***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar"); +***REMOVED*** System.out.println(aa.getName()); +***REMOVED*** System.out.println(aa.getPath()); + + + + + + + ***REMOVED***a.copyFromDmToSVN(aa); +***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "***REMOVED***"))***REMOVED*** +***REMOVED*** System.out.println("AAA");***REMOVED*** +***REMOVED*** +***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar"))***REMOVED*** +***REMOVED*** System.out.println("BBBB"); +***REMOVED*** +***REMOVED******REMOVED*** +***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar"))***REMOVED*** +***REMOVED*** System.out.println("CCCC");***REMOVED*** +***REMOVED*** +***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); +***REMOVED*** a.copyFromDmToSVN(aa, "Dev"); + +***REMOVED*** + + +***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", +***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548")); + +***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", + ***REMOVED*** "***REMOVED***")); + +System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org", + "***REMOVED***")); + + +***REMOVED***Algorithm aa = new Algorithm(); +***REMOVED***aa.setName("UDPIPE_WRAPPER"); +***REMOVED***System.out.println(a.algoExists(aa)); +***REMOVED******REMOVED*** +***REMOVED***ServiceConfiguration bp = new ServiceConfiguration(); +***REMOVED******REMOVED*** +***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***"); +***REMOVED******REMOVED*** +***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); ***REMOVED*** +***REMOVED***System.out.println("ciao"); +***REMOVED*** +***REMOVED******REMOVED*** + +***REMOVED*** +***REMOVED***Algorithm al = new Algorithm(); +***REMOVED*** al.setName("UDPIPE_WRAPPER"); +***REMOVED*** a.deleteFiles(al); + + + + + +***REMOVED*** +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java new file mode 100755 index 0000000..b6b4c3b --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java @@ -0,0 +1,288 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; + +import java.io.File; + +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +***REMOVED*** +***REMOVED*** + +import com.jcraft.jsch.Channel; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.SftpException; + +public class CheckMethodStaging extends CheckMethod***REMOVED*** + + private Logger logger; + + public CheckMethodStaging() + ***REMOVED*** + this.logger = LoggerFactory.getLogger(CheckMethodStaging.class); +***REMOVED*** + + + +***REMOVED*** + public boolean algoExists(Algorithm algo/*, String env*/) throws Exception***REMOVED*** + + this.logger.debug("Looking if algo "+algo.getName()+ " exists"); + Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); + File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar"); + File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar"); + this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath()); + + System.out.println("First file is located to: "+file.getPath()); + System.out.println("Second file is located to: "+file2.getPath()); + + + if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/))) + ***REMOVED*** + this.logger.debug("Files found"); + this.copyFromDmToSVN(file/*,env*/); + this.copyFromDmToSVN(file2/*,env*/); + System.out.println("Files have been copied to SVN"); + + + return true; + ***REMOVED*** + else + ***REMOVED*** + this.logger.debug("Files not found"); + System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); + return false; + ***REMOVED*** + +***REMOVED*** + + + +***REMOVED*** + public void deleteFiles(Algorithm a/*,String env*/) throws Exception***REMOVED*** + JSch jsch = new JSch(); + Session session = null; + Channel channel = null; + ChannelSftp c = null; + Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); + System.out.println("checking existing in env: " + stagingConfiguration.getHost()); + + File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar"); + File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); + + + System.out.println("First file is located to: "+file.getPath()); + System.out.println("Second file is located to: "+file2.getPath()); + + + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + session = jsch.getSession("root", stagingConfiguration.getHost()); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + channel = session.openChannel("sftp"); + channel.connect(); + System.out.println("shell channel connected...."); + + c = (ChannelSftp) channel; + + if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/)))***REMOVED*** + + c.rm(file.getPath()); + c.rm(file2.getPath()); + System.out.println("Both the files have been deleted"); + ***REMOVED*** + else System.out.println("Files not found"); + channel.disconnect(); + c.disconnect(); + session.disconnect(); + +***REMOVED*** + + +***REMOVED*** + public boolean doesExist(String path/*, String env*/) throws Exception ***REMOVED*** + this.logger.debug("Looking if file "+path + " exists"); + JSch jsch = new JSch(); + Session session = null; + Channel channel = null; + ChannelSftp c = null; + boolean success = false; + Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + this.logger.debug("Staging configuration host "+stagingConfiguration.getHost()); + session = jsch.getSession("root",stagingConfiguration.getHost() ); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + channel = session.openChannel("sftp"); + channel.connect(); + System.out.println("shell channel connected...."); + + c = (ChannelSftp) channel; + + ***REMOVED***SftpATTRS is = null; + System.out.println(path); + + try ***REMOVED*** + c.lstat(path); + this.logger.debug("File found"); + success = true; + ***REMOVED*** catch (SftpException e) + ***REMOVED*** + this.logger.error("File not found",e); + + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) + ***REMOVED*** + ***REMOVED*** file doesn't exist + success = false; + ***REMOVED*** + ***REMOVED***success = true; ***REMOVED*** something else went wrong + ***REMOVED*** + channel.disconnect(); + c.disconnect(); + session.disconnect(); + this.logger.debug("Operation result "+success); + return success; + +***REMOVED*** + + +***REMOVED*** + public void copyFromDmToSVN(File a/*,String env*/) throws Exception ***REMOVED*** + JSch jsch = new JSch(); + Session session = null; + SVNUpdater svnUpdater = new SVNUpdaterStaging(); + Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); + + + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + session = jsch.getSession("root", stagingConfiguration.getHost()); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + Channel channel = session.openChannel("sftp"); + channel.connect(); + + ChannelSftp sftp = (ChannelSftp) channel; + + sftp.cd(stagingConfiguration.getGhostAlgoDirectory()); + + System.out.println("REMOTE : "+stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()); + System.out.println("LOCAL : /tmp/"+a.getName()); + + sftp.get(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName()); + + channel.disconnect(); + session.disconnect(); + + File f = new File("/tmp/"+a.getName()); + svnUpdater.updateAlgorithmFiles(f); + f.delete(); +***REMOVED*** + + + + + + public static void main(String[] args) throws Exception ***REMOVED*** +***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); +***REMOVED*** System.out.println(a.getStagingHost()); + + CheckMethodStaging a = new CheckMethodStaging(); + + ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); + +***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar"); +***REMOVED*** System.out.println(aa.getName()); +***REMOVED*** System.out.println(aa.getPath()); + + + + + + + ***REMOVED***a.copyFromDmToSVN(aa); +***REMOVED*** if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "***REMOVED***"))***REMOVED*** +***REMOVED*** System.out.println("AAA");***REMOVED*** +***REMOVED*** +***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar"))***REMOVED*** +***REMOVED*** System.out.println("BBBB"); +***REMOVED*** +***REMOVED******REMOVED*** +***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE_interface.jar"))***REMOVED*** +***REMOVED*** System.out.println("CCCC");***REMOVED*** +***REMOVED*** +***REMOVED*** File aa = new File("/home/gcube/wps_algorithms/algorithms/RBLACKBOX_interface.jar"); +***REMOVED*** a.copyFromDmToSVN(aa, "Dev"); + +***REMOVED*** + + +***REMOVED***System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", +***REMOVED*** "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548")); + +***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", + ***REMOVED*** "***REMOVED***")); + +System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org", + "***REMOVED***")); + + +***REMOVED***Algorithm aa = new Algorithm(); +***REMOVED***aa.setName("UDPIPE_WRAPPER"); +***REMOVED***System.out.println(a.algoExists(aa)); +***REMOVED******REMOVED*** +***REMOVED***ServiceConfiguration bp = new ServiceConfiguration(); +***REMOVED******REMOVED*** +***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***"); +***REMOVED******REMOVED*** +***REMOVED***if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); ***REMOVED*** +***REMOVED***System.out.println("ciao"); +***REMOVED*** +***REMOVED******REMOVED*** + +***REMOVED*** +***REMOVED***Algorithm al = new Algorithm(); +***REMOVED*** al.setName("UDPIPE_WRAPPER"); +***REMOVED*** a.deleteFiles(al); + + + + + +***REMOVED*** +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java new file mode 100755 index 0000000..694e97a --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderProduction.java @@ -0,0 +1,14 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; + +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; + +public class ClusterBuilderProduction extends ClusterBuilder***REMOVED*** + + public ClusterBuilderProduction() ***REMOVED*** + super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); + +***REMOVED*** + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java new file mode 100755 index 0000000..662250d --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/ClusterBuilderStaging.java @@ -0,0 +1,13 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; + +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; + +public class ClusterBuilderStaging extends ClusterBuilder***REMOVED*** + + public ClusterBuilderStaging() ***REMOVED*** + super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); +***REMOVED*** + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java new file mode 100755 index 0000000..8185765 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java @@ -0,0 +1,30 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; + +import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; + +***REMOVED***import scala.actors.threadpool.Arrays; + +public class NotificationHelperProduction extends NotificationHelper***REMOVED*** + +***REMOVED*** private Exception executionException; + + + private String getSubjectHeader() ***REMOVED*** + return "[DataMinerGhostProductionInstallationRequestReport]"; + ***REMOVED*** + + @Override + public String getSuccessSubject() ***REMOVED*** + return this.getSubjectHeader()+" is SUCCESS"; + ***REMOVED*** + + + + @Override + public String getFailedSubject() ***REMOVED*** + return String.format(this.getSubjectHeader()+" is FAILED"); + ***REMOVED*** + + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java new file mode 100755 index 0000000..c8752cc --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java @@ -0,0 +1,32 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; + +import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; + +***REMOVED***import scala.actors.threadpool.Arrays; + +public class NotificationHelperStaging extends NotificationHelper ***REMOVED*** + +***REMOVED*** private Exception executionException; + + + + private String getSubjectHeader() ***REMOVED*** + return "[DataMinerGhostStagingInstallationRequestReport]"; + ***REMOVED*** + + + @Override + public String getSuccessSubject() ***REMOVED*** + return this.getSubjectHeader()+" is SUCCESS"; + ***REMOVED*** + + + + + @Override + public String getFailedSubject() ***REMOVED*** + return String.format(this.getSubjectHeader()+" is FAILED"); + ***REMOVED*** + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java new file mode 100755 index 0000000..3b7918f --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterProduction.java @@ -0,0 +1,21 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; + +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +***REMOVED*** + +/** + * Created by ggiammat on 5/9/17. + */ +public class SVNUpdaterProduction extends SVNUpdater***REMOVED*** + + + + + public SVNUpdaterProduction() throws SVNException ***REMOVED*** + super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); +***REMOVED*** + + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java new file mode 100755 index 0000000..79d405b --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/SVNUpdaterStaging.java @@ -0,0 +1,21 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; + +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +***REMOVED*** + +/** + * Created by ggiammat on 5/9/17. + */ +public class SVNUpdaterStaging extends SVNUpdater***REMOVED*** + + + public SVNUpdaterStaging() throws SVNException ***REMOVED*** + super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); +***REMOVED*** + + + + + +***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java index ffe0aae..2d40976 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java @@ -6,6 +6,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager; ***REMOVED*** ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterProduction; ***REMOVED*** public class JobTest ***REMOVED*** @@ -14,7 +15,7 @@ public class JobTest ***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); - SVNUpdater svnUpdater = new SVNUpdater(); + SVNUpdater svnUpdater = new SVNUpdaterProduction(); Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0"); ***REMOVED***test phase diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java index 66d0099..eccc491 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -5,6 +5,7 @@ package org.gcube.dataanalysis.dataminerpoolmanager; ***REMOVED*** ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.gcube.dataanalysis.dataminer.poolmanager.util.impl.SVNUpdaterStaging; ***REMOVED*** /** @@ -17,7 +18,7 @@ public class SVNTests ***REMOVED*** public static void main(String[] args) throws SVNException, IOException, InterruptedException ***REMOVED*** - SVNUpdater svnUpdater = new SVNUpdater(); + SVNUpdater svnUpdater = new SVNUpdaterStaging(); Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); ***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); @@ -27,7 +28,7 @@ public class SVNTests ***REMOVED*** ***REMOVED***System.out.println(algo.getCategory()); ***REMOVED***System.out.println(algo.getSkipJava()); - svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), ""); + ***REMOVED***svnUpdater.updateSVNAlgorithmList("/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms", algo, "/gcube/devNext/NextNext", algo.getCategory(), algo.getAlgorithmType(), ""); ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); ***REMOVED***svnUpdater.readRPRotoDeps(algo);