From c40cec37332d43d29bd8fd13a20edfaad236bac5 Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Wed, 1 Mar 2017 15:21:40 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@144486 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../ansiblebridge/AnsibleBridge.java | 9 +- .../poolmanager/clients/ISClient.java | 3 +- .../process/AlgorithmPackageParser.java | 103 +++++++++++++++--- .../poolmanager/rest/RestPoolManager.java | 99 ++++++++++------- 4 files changed, 155 insertions(+), 59 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index b4678ca..e7bc5db 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -203,11 +203,12 @@ public class AnsibleBridge { FileOutputStream fos = new FileOutputStream(n); PrintStream ps = new PrintStream(fos); System.setOut(ps); - System.setErr(ps); - worker.apply(); - System.setOut(console); + ***REMOVED***System.setErr(ps); System.setErr(console); + worker.apply(); + ***REMOVED***System.setOut(console); + ***REMOVED***worker.apply(); System.out.println("Log stored to to " + n.getAbsolutePath()); ***REMOVED*** destroy the worker @@ -235,6 +236,8 @@ public class AnsibleBridge { */ public Collection generateRoles(Dependency d) { Collection roles = new Vector<>(); + + if("os".equalsIgnoreCase(d.getType())) { OSDependencyPackage pkg = new OSDependencyPackage(d); if(pkg!=null) { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index 1afbfba..3a48717 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -30,7 +30,8 @@ public class ISClient { Collection out = new Vector<>(); Host h = new Host(); ***REMOVED***h.setName("bb-dataminer.res.eng.it"); - h.setName("vm101.ui.savba.sk"); + ***REMOVED***h.setName("vm101.ui.savba.sk"); + h.setName("dataminer1-devnext.d4science.org"); out.add(h); return out; ***REMOVED*** else { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java index 41f4b47..7fcfd70 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java @@ -10,6 +10,7 @@ import java.util.Vector; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; +import org.apache.commons.lang.StringEscapeUtils; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; @@ -149,24 +150,91 @@ public class AlgorithmPackageParser { * @param metadata * @return */ - private Algorithm createAlgorithm(Map> metadata) { - Algorithm out = new Algorithm(); - out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); - out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); - out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); - List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); - if (dependencies != null) { - for (String pkg : dependencies) { - Dependency dep = new Dependency(); - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** +***REMOVED*** private Algorithm createAlgorithm(Map> metadata) { +***REMOVED*** Algorithm out = new Algorithm(); +***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); +***REMOVED*** out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); +***REMOVED*** out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); +***REMOVED*** List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); +***REMOVED*** if (dependencies != null) { +***REMOVED*** for (String pkg : dependencies) { +***REMOVED*** Dependency dep = new Dependency(); +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("os"); +***REMOVED*** out.addDependency(dep); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** return out; +***REMOVED*** ***REMOVED*** + private Algorithm createAlgorithm(Map> metadata) { + Algorithm out = new Algorithm(); + out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); + out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); + out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); + List rdependencies = extractMultipleValues(metadata, "r"); + if (rdependencies != null) { + for (String pkg : rdependencies) { + Dependency dep = new Dependency(); + + ***REMOVED***if (pkg.startsWith("os:")){ + dep.setName(pkg); + dep.setType("cran"); + out.addDependency(dep); + ***REMOVED*** + ***REMOVED*** + + + List osdependencies = extractMultipleValues(metadata, "os"); + if (osdependencies != null) { + for (String pkg : osdependencies) { + Dependency dep = new Dependency(); + + ***REMOVED***if (pkg.startsWith("os:")){ + dep.setName(pkg); + dep.setType("os"); + out.addDependency(dep); + ***REMOVED*** + ***REMOVED*** + + + + List cdependencies = extractMultipleValues(metadata, "custom"); + if (cdependencies != null) { + for (String pkg : cdependencies) { + Dependency dep = new Dependency(); + + ***REMOVED***if (pkg.startsWith("os:")){ + dep.setName(pkg); + dep.setType("custom"); + out.addDependency(dep); + ***REMOVED*** + ***REMOVED*** + + +***REMOVED*** if (pkg.startsWith("r:")){ +***REMOVED*** ***REMOVED***String results = StringEscapeUtils.escapeJava(pkg); +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("cran"); +***REMOVED*** ***REMOVED*** +***REMOVED*** if (pkg.startsWith("custom:")){ +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("custom"); +***REMOVED*** ***REMOVED*** +***REMOVED*** if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:")){ +***REMOVED*** dep.setName(pkg); +***REMOVED*** dep.setType("os"); +***REMOVED*** ***REMOVED*** + + + + return out; + ***REMOVED*** + + + + private static String extractSingleValue(Map> metadata, String key) { List l = metadata.get(key); @@ -177,6 +245,9 @@ public class AlgorithmPackageParser { ***REMOVED*** ***REMOVED*** + + + private static List extractMultipleValues( Map> metadata, String key) { List l = metadata.get(key); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index 7ea5967..b796fcc 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -1,8 +1,8 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest; -import java.io.File; ***REMOVED*** import java.net.MalformedURLException; +import java.net.ProxySelector; ***REMOVED*** import java.net.UnknownHostException; @@ -13,12 +13,14 @@ import javax.ws.rs.QueryParam; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminerpoolmanager.AlgorithmPackageParserTest; +import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; ***REMOVED*** ***REMOVED*** + + + @Path("/") public class RestPoolManager implements PoolManager { @@ -26,52 +28,46 @@ public class RestPoolManager implements PoolManager { private PoolManager service = new DataminerPoolManager(); + @GET @Path("/add") @Produces("text/plain") - public String addAlgorithmToVRE(@QueryParam("algorithm") String algorithm, @QueryParam("vre") String vre, - @QueryParam("name") String name, @QueryParam("description") String description, - @QueryParam("category") String category, @QueryParam("algorithmType") String algorithmType, + public String addAlgorithmToVRE( + @QueryParam("algorithm") String algorithm, + @QueryParam("vre") String vre, + @QueryParam("name") String name, + @QueryParam("description") String description, + @QueryParam("category") String category, + @QueryParam("algorithmType") String algorithmType, @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException { ***REMOVED*** TODO Auto-generated method stub LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre); Algorithm algo = service.extractAlgorithm(algorithm); - - if (algo.getCategory() == null) { + + if (algo.getCategory() == null){ algo.setCategory(category); - ***REMOVED*** else - category = algo.getCategory(); - System.out.println("category is: " + category); - - if (algo.getAlgorithmType() == null) { + ***REMOVED*** else category = algo.getCategory(); + + if (algo.getAlgorithmType() == null){ algo.setAlgorithmType(algorithmType); - ***REMOVED*** else - algorithmType = algo.getCategory(); - System.out.println("algorithmType is: " + algorithmType); - - if (algo.getSkipJava() == null) { + ***REMOVED*** else algorithmType = algo.getCategory(); + + if (algo.getSkipJava() == null){ algo.setSkipJava(skipJava); - ***REMOVED*** else - skipJava = algo.getSkipJava(); - System.out.println("skipJava is: " + skipJava); - - if (algo.getName() == null) { + ***REMOVED*** else skipJava = algo.getSkipJava(); + + if (algo.getName() == null){ algo.setCategory(name); - ***REMOVED*** else - name = algo.getName(); - System.out.println("name is: " + name); - - if (algo.getDescription() == null) { - algo.setDescription(description); - ; - ***REMOVED*** else - description = algo.getDescription(); - System.out.println("description is: " + description); + ***REMOVED*** else name = algo.getName(); + + if (algo.getDescription() == null){ + algo.setDescription(description);; + ***REMOVED*** else description = algo.getDescription(); return service.addAlgorithmToVRE(algo, vre); - ***REMOVED*** + @GET @Path("/log") @Produces("text/plain") @@ -80,6 +76,10 @@ public class RestPoolManager implements PoolManager { LOGGER.debug("Returning Log =" + logUrl); return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl)); ***REMOVED*** + + + + @Override public Algorithm extractAlgorithm(String url) throws IOException { @@ -87,37 +87,58 @@ public class RestPoolManager implements PoolManager { return null; ***REMOVED*** + public static void main(String[] args) throws IOException, InterruptedException { RestPoolManager a = new RestPoolManager(); - ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); + ***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); + + + ***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); + ScopeProvider.instance.set("/gcube/devNext/NextNext"); - a.addAlgorithmToVRE("http:***REMOVED***data.d4science.org/am84enRYYkZ3dGRjZk9qTytQTndqaFpTOFJtT3RQeGhHbWJQNStIS0N6Yz0", - "/gcube/devNext/NextNext", null, null, "ICHTHYOP_MODEL", "transducerers", "N"); - ***REMOVED*** System.out.println(a.getLogById("dadcb059-69e5-48c3-aa58-3b290ae0419d")); + a.addAlgorithmToVRE( + "http:***REMOVED***data.d4science.org/cXhJV0s4b3dUN05jZk9qTytQTndqdmVUdURLOTRHeHpHbWJQNStIS0N6Yz0", + "/gcube/devNext/NextNext", + null, null, "test", "transducerers", "N"); + + ***REMOVED***System.out.println(a.getLogById("34ac474d-b9df-4929-87e1-2a0ae26cf898")); ***REMOVED*** + @Override public void getLogId(Algorithm algo, String vre) { ***REMOVED*** TODO Auto-generated method stub - + ***REMOVED*** + @Override public String getScriptFromURL(URL logId) throws IOException { ***REMOVED*** TODO Auto-generated method stub return null; ***REMOVED*** + @Override public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException { ***REMOVED*** TODO Auto-generated method stub return null; ***REMOVED*** + @Override public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { ***REMOVED*** TODO Auto-generated method stub return null; ***REMOVED*** + + + + + + + + + ***REMOVED*** \ No newline at end of file