From 536e5679e3e3cacbba0a1451b59b68157a72c00d Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Thu, 13 Jul 2017 16:13:18 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@151067 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../poolmanager/ansible/AnsibleWorker.java | 4 +- .../poolmanager/service/DMPMJob.java | 10 ++- .../service/DataminerPoolManager.java | 14 ++++ .../poolmanager/service/StagingJob.java | 82 +++++++++++++++---- .../poolmanager/util/CheckMethod.java | 4 +- .../dataminer/poolmanager/util/SendMail.java | 2 +- .../dataminerpoolmanager/SVNTests.java | 9 +- 7 files changed, 101 insertions(+), 24 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index df5ff0e..7c05031 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -8,6 +8,7 @@ import java.io.File; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; +import java.io.PrintWriter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -118,7 +119,6 @@ public class AnsibleWorker ***REMOVED*** ***REMOVED*** - public int execute(PrintStream ps) throws IOException, InterruptedException, SVNException ***REMOVED*** @@ -130,6 +130,8 @@ public class AnsibleWorker ***REMOVED*** inheritIO(p.getInputStream(), ps); inheritIO(p.getErrorStream(), ps); +***REMOVED*** writer.println(this.getStatus(p.waitFor())); +***REMOVED*** writer.close(); return p.waitFor(); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java index 6a03a3f..4aa36bd 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.FileOutputStream; ***REMOVED*** import java.io.PrintStream; +import java.io.PrintWriter; import java.util.LinkedList; ***REMOVED*** import java.util.UUID; @@ -16,6 +17,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; ***REMOVED*** +import Jampack.Print; + public abstract class DMPMJob ***REMOVED*** @@ -69,9 +72,14 @@ public abstract class DMPMJob ***REMOVED*** protected int executeAnsibleWorker(AnsibleWorker worker) throws IOException, InterruptedException, SVNException***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "jobs"); path.mkdirs(); + File n = new File(this.jobLogs + File.separator + this.id); FileOutputStream fos = new FileOutputStream(n, true); - PrintStream ps = new PrintStream(fos); + PrintStream ps = new PrintStream(fos); + +***REMOVED*** File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); +***REMOVED*** PrintWriter fos2 = new PrintWriter(m, "UTF-8"); + return worker.execute(ps); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 91ce431..ea4fa55 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -56,4 +56,18 @@ public class DataminerPoolManager ***REMOVED*** return new Scanner(path).useDelimiter("\\Z").next(); ***REMOVED*** + + + public String getMonitorById(String id) throws FileNotFoundException***REMOVED*** + + ***REMOVED***TODO: load dir from configuration file + File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/jobs/" + + id + "_exitStatus"); + + return new Scanner(path).useDelimiter("\\Z").next(); +***REMOVED*** + + + + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index a44951d..d230881 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -2,6 +2,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; import static org.gcube.common.authorization.client.Constants.authorizationService; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.io.UnsupportedEncodingException; import java.util.Collection; import org.gcube.common.authorization.library.AuthorizationEntry; @@ -19,15 +23,20 @@ public class StagingJob extends DMPMJob ***REMOVED*** private Algorithm algorithm; private Cluster stagingCluster; - ***REMOVED***private Cluster rProtoCluster; + ***REMOVED*** private Cluster rProtoCluster; private String rProtoVREName; - public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, /*Cluster rProtoCluster,*/ + public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, + Cluster stagingCluster, /* Cluster rProtoCluster, */ String rProtoVREName) ***REMOVED*** super(svnUpdater); + this.jobLogs = new File( + System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs"); + this.jobLogs.mkdirs(); + this.algorithm = algorithm; this.stagingCluster = stagingCluster; - ***REMOVED***this.rProtoCluster = rProtoCluster; + ***REMOVED*** this.rProtoCluster = rProtoCluster; this.rProtoVREName = rProtoVREName; ***REMOVED*** @@ -41,7 +50,8 @@ public class StagingJob extends DMPMJob ***REMOVED*** try ***REMOVED*** Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies( - this.svnUpdater.getRProtoDependencyFile(this.algorithm.getLanguage()), this.algorithm.getDependencies()); + this.svnUpdater.getRProtoDependencyFile(this.algorithm.getLanguage()), + this.algorithm.getDependencies()); if (!undefinedDependencies.isEmpty()) ***REMOVED*** @@ -56,27 +66,67 @@ public class StagingJob extends DMPMJob ***REMOVED*** int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); + Integer s = null; + s = Integer.valueOf(ret); + + if (s == null) ***REMOVED*** + this.getStatus(0); + ***REMOVED*** + + if (ret != 0) ***REMOVED*** + this.getStatus(2); + sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("Installation failed. Return code=" + ret)); + ***REMOVED*** + if (ret == 0) ***REMOVED*** + this.getStatus(0); if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get()) && (b.algoExists(this.algorithm))) ***REMOVED*** - - sm.sendNotification(nh.getSuccessSubject(), nh.getSuccessBody()); - ***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm); - + this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, this.algorithm.getFullname(), "Proto"); - - ***REMOVED*** int ret2 = this.executeAnsibleWorker( - ***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, - ***REMOVED*** false, "gcube")); - ***REMOVED*** - ***REMOVED*** else - sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("Installation failed. Return code=" + ret)); - + + this.getStatus(9); + sm.sendNotification(nh.getSuccessSubject(), nh.getSuccessBody()); + ***REMOVED*** else + this.getStatus(2); + sm.sendNotification(nh.getFailedSubject(), + nh.getFailedBody("DataMiner Interface not working or files " + this.algorithm.getName() + + ".jar and " + this.algorithm.getName() + + "_interface.jar not available at the expected path")); + ***REMOVED*** ***REMOVED*** catch (Exception e) ***REMOVED*** e.printStackTrace(); ***REMOVED*** ***REMOVED*** + public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException ***REMOVED*** + + File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); + + PrintWriter writer = new PrintWriter(m, "UTF-8"); + + String response = ""; + + if (exitstatus == 0) ***REMOVED*** + response = "IN PROGRESS"; + writer.println(response); + writer.close(); + ***REMOVED*** + + if (exitstatus == 9) ***REMOVED*** + response = "COMPLETED"; + writer.println(response); + writer.close(); + ***REMOVED*** + + if (exitstatus == 2) ***REMOVED*** + response = "FAILED"; + writer.println(response); + writer.close(); + ***REMOVED*** + return response; +***REMOVED*** + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java index 286c643..6d40fa7 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java @@ -71,8 +71,8 @@ public class CheckMethod ***REMOVED*** public boolean algoExists(Algorithm a)***REMOVED*** - File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar"); - File file2 = new File(" /home/gcube/wps_algorithms/algorithms/"+a.getName()+"interface_.jar"); + File file = new File(System.getProperty("user.home")+File.separator+"wps_algorithms/algorithms/"+a.getName()+".jar"); + File file2 = new File(System.getProperty("user.home")+File.separator+"wps_algorithms/algorithms/"+a.getName()+"interface_.jar"); if (file.exists()&&(file2.exists()))***REMOVED*** return true; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java index 977790f..3094d3b 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -107,7 +107,7 @@ public class SendMail extends StandardLocalInfraAlgorithm ***REMOVED*** ***REMOVED***String serviceAddress = ep.getResourceEntyName();***REMOVED***"https:***REMOVED***socialnetworking1.d4science.org/social-networking-library-ws/rest/"; serviceAddress = serviceAddress.endsWith("/") ? serviceAddress : serviceAddress + "/"; serviceAddress+= "2/users/get-usernames-by-global-role?"; - serviceAddress+= "role-name=Administrator" + "&gcube-token=" + SecurityTokenProvider.instance.get(); + serviceAddress+= "role-name=DataMiner-Manager" + "&gcube-token=" + SecurityTokenProvider.instance.get(); System.out.println("Url is " + serviceAddress); diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java index 8c6bf9e..e6ef618 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -19,10 +19,13 @@ public class SVNTests ***REMOVED*** SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ngalante/workspace/dataminer-pool-manager/src/main/resources/service.properties")); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/R0ExYjFPVFBrOUlNdEhrTUQyQlZjbUJuQ20rbGFrZ0pHbWJQNStIS0N6Yz0"); + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0"); ***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); - + System.out.println(algo.getAlgorithmType()); + System.out.println(algo.getCategory()); + System.out.println(algo.getSkipJava()); + ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto"); - svnUpdater.readRPRotoDeps(algo); + ***REMOVED***svnUpdater.readRPRotoDeps(algo); ***REMOVED*** ***REMOVED***