From ac0ae80a9e57ec1ff36ed862acfd524ab5857353 Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Wed, 29 Mar 2017 14:06:46 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146429 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 6 + .../poolmanager/datamodel/AlgoResource.java | 22 + .../poolmanager/datamodel/Algorithm.java | 29 +- .../poolmanager/rest/PoolManager.java | 9 +- .../poolmanager/rest/RestPoolManager.java | 56 +- .../service/DataminerPoolManager.java | 634 +++++++++++------- 6 files changed, 481 insertions(+), 275 deletions(-) create mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java diff --git a/pom.xml b/pom.xml index 47b739d..f7fa8d7 100644 --- a/pom.xml +++ b/pom.xml @@ -24,6 +24,12 @@ + + org.gcube.core + common-smartgears + + provided + commons-io commons-io diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java new file mode 100644 index 0000000..c899055 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgoResource.java @@ -0,0 +1,22 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; + +public class AlgoResource { + + protected String id; + + public AlgoResource() { +***REMOVED*** + + public AlgoResource(String id) { + this.id = id; +***REMOVED*** + + public String getId() { + return this.id; +***REMOVED*** + + public void setId(String id) { + this.id = id; +***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java index 3d85ba7..f9fbe88 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java @@ -3,8 +3,35 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; import java.util.Collection; import java.util.Vector; -public class Algorithm { +public class Algorithm extends AlgoResource { + + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((id == null) ? 0 : id.hashCode()); + return result; +***REMOVED*** + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + Algorithm other = (Algorithm) obj; + if (id == null) { + if (other.id != null) + return false; + ***REMOVED*** else if (!id.equals(other.id)) + return false; + return true; +***REMOVED*** + private String name; private String description; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index be1b1ec..e1ddc17 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -31,6 +31,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest; import java.net.MalformedURLException; ***REMOVED*** import java.net.UnknownHostException; +import java.util.Set; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; @@ -47,6 +48,12 @@ public interface PoolManager { String getScriptFromURL(URL logId) throws IOException; - URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; + URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; + + void addAlgToIs(Algorithm algo); + + Set getAlgoFromIs(); + + Algorithm getAlgoById(String id); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index 4ea9712..11218c9 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -147,12 +147,16 @@ import java.net.MalformedURLException; import java.net.ProxySelector; ***REMOVED*** import java.net.UnknownHostException; +import java.util.Random; +import java.util.Set; +import java.util.UUID; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +import org.apache.commons.lang.math.RandomUtils; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; @@ -175,7 +179,7 @@ public class RestPoolManager implements PoolManager { /hosts/ POST ***REMOVED*** add an algorithm to the given host */ - @GET + @GET @Path("/scopes/add") @Produces("text/plain") public String addAlgorithmToVRE( @@ -187,6 +191,7 @@ public class RestPoolManager implements PoolManager { @QueryParam("algorithmType") String algorithmType, @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException { Algorithm algo= this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava); + ***REMOVED***service.addAlgToIs(algo); return service.addAlgorithmToVRE(algo, vre); ***REMOVED*** @@ -202,6 +207,7 @@ public class RestPoolManager implements PoolManager { @QueryParam("algorithmType") String algorithmType, @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException { Algorithm algo= this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, skipJava); + service.addAlgToIs(algo); return service.addAlgorithmToHost(algo, hostname); ***REMOVED*** @@ -218,6 +224,7 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** TODO Auto-generated method stub LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre); Algorithm algo = service.extractAlgorithm(algorithm); + algo.setId(name+"@"+UUID.randomUUID()); if (algo.getCategory() == null){ algo.setCategory(category); @@ -303,28 +310,30 @@ public class RestPoolManager implements PoolManager { public static void main(String[] args) throws IOException, InterruptedException { - RestPoolManager a = new RestPoolManager(); ***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); - - ***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); ScopeProvider.instance.set("/gcube/devNext/NextNext"); -***REMOVED*** a.addAlgorithmToVRE( -***REMOVED*** "http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0", -***REMOVED*** "/gcube/devNext/NextNext", -***REMOVED*** null, null, "test", "transducerers", "N"); + ***REMOVED***PoolManager aa = new DataminerPoolManager(); + ***REMOVED***System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3")); + ***REMOVED***System.out.println(aa.getAlgoFromIs()); + + RestPoolManager a = new RestPoolManager(); + ***REMOVED***a.addAlgorithmToVRE( + ***REMOVED***"http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0", + ***REMOVED***"/gcube/devNext/NextNext", + ***REMOVED***null, null, "test", "transducerers", "N"); + a.addAlgorithmToHost( - "http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0", - "dataminer1-devnext.d4science.org", - null, null, "test", "transducerers", "N"); - - ***REMOVED***System.out.println(a.getLogById("34ac474d-b9df-4929-87e1-2a0ae26cf898")); + "http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0", + "dataminer1-devnext.d4science.org", + "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N"); ***REMOVED*** + @Override public void getLogId(Algorithm algo, String vre) { ***REMOVED*** TODO Auto-generated method stub @@ -358,6 +367,27 @@ public class RestPoolManager implements PoolManager { return null; ***REMOVED*** + @Override + public void addAlgToIs(Algorithm algo) { + ***REMOVED*** TODO Auto-generated method stub + +***REMOVED*** + + + + + @Override + public Set getAlgoFromIs() { + ***REMOVED*** TODO Auto-generated method stub + return null; +***REMOVED*** + + @Override + public Algorithm getAlgoById(String id) { + ***REMOVED*** TODO Auto-generated method stub + return null; +***REMOVED*** + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index bec08cc..aa13112 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -262,16 +262,29 @@ ***REMOVED******REMOVED*** package org.gcube.dataanalysis.dataminer.poolmanager.service; +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + ***REMOVED*** import java.io.File; ***REMOVED*** ***REMOVED*** +import java.io.StringWriter; import java.net.MalformedURLException; ***REMOVED*** import java.net.URLConnection; import java.net.UnknownHostException; +import java.util.Arrays; +import java.util.HashSet; +***REMOVED*** +import java.util.Set; import java.util.UUID; +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.HostingNode; +import org.gcube.common.resources.gcore.Resources; +import org.gcube.common.resources.gcore.Software.Profile.Dependency; +***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; @@ -280,289 +293,390 @@ import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager; +import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; +import org.gcube.informationsystem.publisher.RegistryPublisherFactory; +import org.gcube.informationsystem.publisher.ScopedPublisher; +import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; +import org.gcube.resources.discovery.client.api.DiscoveryClient; +import org.gcube.resources.discovery.client.queries.api.SimpleQuery; + ***REMOVED*** ***REMOVED*** public class DataminerPoolManager implements PoolManager { - private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); - -***REMOVED*** static Collection algorithms; -***REMOVED*** -***REMOVED*** static Collection sets; -***REMOVED*** -***REMOVED*** static { -***REMOVED*** algorithms = new Vector<>(); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public DataminerPoolManager() { -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected -***REMOVED*** * on the pool. -***REMOVED*** */ -***REMOVED*** public void publishAlgorithm(Algorithm algorithm) { -***REMOVED*** algorithms.add(algorithm); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Re-deploy the given algorithm wherever it's installed -***REMOVED*** * -***REMOVED*** * @param algorithm -***REMOVED*** */ -***REMOVED*** /* -***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement this ***REMOVED*** -***REMOVED*** */ -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Add the give algorithm to the given set -***REMOVED*** * -***REMOVED*** * @param algorithmId -***REMOVED*** * @param setId -***REMOVED*** */ -***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) { -***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); -***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName); -***REMOVED*** if (set != null && algorithm != null) { -***REMOVED*** set.addAlgorithm(algorithm); -***REMOVED*** this.updateClusters(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Apply the given set of algorithms to the given cluster -***REMOVED*** * -***REMOVED*** * @param setId -***REMOVED*** * @param clusterId -***REMOVED*** */ -***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) { -***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); -***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName); -***REMOVED*** if (set != null && cluster != null) { -***REMOVED*** cluster.addAlgorithmSet(set); -***REMOVED*** this.updateClusters(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) { -***REMOVED*** for (AlgorithmSet set : sets) { -***REMOVED*** if (name.equals(set.getName())) { -***REMOVED*** return set; -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** private Algorithm getAlgorithm(String name) { -***REMOVED*** for (Algorithm a : algorithms) { -***REMOVED*** if (name.equals(a.getName())) { -***REMOVED*** return a; -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** + ***REMOVED*** static Collection algorithms; + ***REMOVED*** + ***REMOVED*** static Collection sets; + ***REMOVED*** + ***REMOVED*** static { + ***REMOVED*** algorithms = new Vector<>(); + ***REMOVED*** ***REMOVED*** + ***REMOVED*** + ***REMOVED*** public DataminerPoolManager() { + ***REMOVED*** ***REMOVED*** + ***REMOVED*** + ***REMOVED*** /** + ***REMOVED*** * Add a new algorithm to the set of known ones. No further action is + ***REMOVED*** expected + ***REMOVED*** * on the pool. + ***REMOVED*** */ + ***REMOVED*** public void publishAlgorithm(Algorithm algorithm) { + ***REMOVED*** algorithms.add(algorithm); + ***REMOVED*** ***REMOVED*** + ***REMOVED*** + ***REMOVED*** /** + ***REMOVED*** * Re-deploy the given algorithm wherever it's installed + ***REMOVED*** * + ***REMOVED*** * @param algorithm + ***REMOVED*** */ + ***REMOVED*** /* + ***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement + ***REMOVED*** this ***REMOVED*** + ***REMOVED*** */ + ***REMOVED*** + ***REMOVED*** /** + ***REMOVED*** * Add the give algorithm to the given set + ***REMOVED*** * + ***REMOVED*** * @param algorithmId + ***REMOVED*** * @param setId + ***REMOVED*** */ + ***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) { + ***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); + ***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName); + ***REMOVED*** if (set != null && algorithm != null) { + ***REMOVED*** set.addAlgorithm(algorithm); + ***REMOVED*** this.updateClusters(); + ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED*** + ***REMOVED*** + ***REMOVED*** /** + ***REMOVED*** * Apply the given set of algorithms to the given cluster + ***REMOVED*** * + ***REMOVED*** * @param setId + ***REMOVED*** * @param clusterId + ***REMOVED*** */ + ***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String + ***REMOVED*** clusterName) { + ***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); + ***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName); + ***REMOVED*** if (set != null && cluster != null) { + ***REMOVED*** cluster.addAlgorithmSet(set); + ***REMOVED*** this.updateClusters(); + ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED*** + ***REMOVED*** + ***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) { + ***REMOVED*** for (AlgorithmSet set : sets) { + ***REMOVED*** if (name.equals(set.getName())) { + ***REMOVED*** return set; + ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED*** + ***REMOVED*** return null; + ***REMOVED*** ***REMOVED*** + ***REMOVED*** + ***REMOVED*** private Algorithm getAlgorithm(String name) { + ***REMOVED*** for (Algorithm a : algorithms) { + ***REMOVED*** if (name.equals(a.getName())) { + ***REMOVED*** return a; + ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED*** + ***REMOVED*** return null; + ***REMOVED*** ***REMOVED*** - + ***REMOVED*** + ***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) { + ***REMOVED*** new Thread() { + ***REMOVED*** public void run() { + ***REMOVED*** while (true) { + ***REMOVED*** try { + ***REMOVED*** addAlgorithmToVRE(algorithm, vre); + ***REMOVED*** ***REMOVED*** catch (Exception e) { + ***REMOVED*** ***REMOVED***log here + ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED***.start(); + ***REMOVED*** ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** public String getLogId(){ + ***REMOVED*** PrintStream console = System.out; + ***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); + ***REMOVED*** path.mkdirs(); + ***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); + ***REMOVED*** FileOutputStream fos = new FileOutputStream(n); + ***REMOVED*** PrintStream ps = new PrintStream(fos); + ***REMOVED*** System.setOut(ps); + ***REMOVED*** worker.apply(); + ***REMOVED*** System.setOut(console); + ***REMOVED*** worker.apply(); + ***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); + ***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) { -***REMOVED*** new Thread() { -***REMOVED*** public void run() { -***REMOVED*** while (true) { -***REMOVED*** try { -***REMOVED*** addAlgorithmToVRE(algorithm, vre); -***REMOVED*** ***REMOVED*** catch (Exception e) { -***REMOVED*** ***REMOVED***log here -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED***.start(); -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** - -***REMOVED*** public String getLogId(){ -***REMOVED*** PrintStream console = System.out; -***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); -***REMOVED*** path.mkdirs(); -***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); -***REMOVED*** FileOutputStream fos = new FileOutputStream(n); -***REMOVED*** PrintStream ps = new PrintStream(fos); -***REMOVED*** System.setOut(ps); -***REMOVED*** worker.apply(); -***REMOVED*** System.setOut(console); -***REMOVED*** worker.apply(); -***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); -***REMOVED******REMOVED*** - - - - -***REMOVED*** public String getLogById(String id) throws IOException { -***REMOVED*** String strLine = null; -***REMOVED*** try{ -***REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id); -***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); -***REMOVED*** /* read log line by line */ -***REMOVED*** while ((strLine = br.readLine()) != null) { -***REMOVED*** /* parse strLine to obtain what you want */ -***REMOVED*** System.out.println (strLine); -***REMOVED*** ***REMOVED*** -***REMOVED*** br.close(); -***REMOVED*** ***REMOVED*** catch (Exception e) { -***REMOVED*** System.err.println("Error: " + e.getMessage()); -***REMOVED*** ***REMOVED*** -***REMOVED*** return strLine; -***REMOVED******REMOVED*** - + ***REMOVED*** public String getLogById(String id) throws IOException { + ***REMOVED*** String strLine = null; + ***REMOVED*** try{ + ***REMOVED*** FileInputStream fstream = new + ***REMOVED*** FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id); + ***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); + ***REMOVED*** /* read log line by line */ + ***REMOVED*** while ((strLine = br.readLine()) != null) { + ***REMOVED*** /* parse strLine to obtain what you want */ + ***REMOVED*** System.out.println (strLine); + ***REMOVED*** ***REMOVED*** + ***REMOVED*** br.close(); + ***REMOVED*** ***REMOVED*** catch (Exception e) { + ***REMOVED*** System.err.println("Error: " + e.getMessage()); + ***REMOVED*** ***REMOVED*** + ***REMOVED*** return strLine; + ***REMOVED*** ***REMOVED*** - public String getScriptFromURL(URL url) throws IOException { - if (url == null) { - return null; - ***REMOVED*** - URLConnection yc = url.openConnection(); - BufferedReader input = new BufferedReader(new InputStreamReader( - yc.getInputStream())); - String line; - StringBuffer buffer = new StringBuffer(); - while ((line = input.readLine()) != null) { - buffer.append(line + "\n"); - ***REMOVED*** - String bufferScript = buffer.substring(0, buffer.length()); - input.close(); - return bufferScript; - ***REMOVED*** - - - - - - /** - * Publish the given algorithm in the given VRE - * - * @param algorithmName - * @param vre - * - */ - public String addAlgorithmToVRE(Algorithm algorithm, final String vre) - throws IOException { - ***REMOVED*** create a fake algorithm set - final AlgorithmSet algoSet = new AlgorithmSet(); - algoSet.setName("fake"); - algoSet.addAlgorithm(algorithm); - final String uuid = UUID.randomUUID().toString(); + public String getScriptFromURL(URL url) throws IOException { + if (url == null) { + return null; + ***REMOVED*** + URLConnection yc = url.openConnection(); + BufferedReader input = new BufferedReader(new InputStreamReader(yc.getInputStream())); + String line; + StringBuffer buffer = new StringBuffer(); + while ((line = input.readLine()) != null) { + buffer.append(line + "\n"); + ***REMOVED*** + String bufferScript = buffer.substring(0, buffer.length()); + input.close(); + return bufferScript; +***REMOVED*** - new Thread(new Runnable() { - @Override - public void run() { - ***REMOVED*** TODO Auto-generated method stub - try { - addAlgorithmsToVRE(algoSet, vre, uuid); - ***REMOVED*** catch (IOException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); - ***REMOVED*** this line will execute immediately, not waiting for your task to complete - System.out.println(uuid); - return uuid; - ***REMOVED*** - - public String addAlgorithmToHost(Algorithm algorithm, final String hostname) - throws IOException { - ***REMOVED*** create a fake algorithm set - final AlgorithmSet algoSet = new AlgorithmSet(); - algoSet.setName("fake"); - algoSet.addAlgorithm(algorithm); - final String uuid = UUID.randomUUID().toString(); + /** + * Publish the given algorithm in the given VRE + * + * @param algorithmName + * @param vre + * + */ + public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException { + ***REMOVED*** create a fake algorithm set + final AlgorithmSet algoSet = new AlgorithmSet(); + algoSet.setName("fake"); + algoSet.addAlgorithm(algorithm); + final String uuid = UUID.randomUUID().toString(); - new Thread(new Runnable() { - @Override - public void run() { - ***REMOVED*** TODO Auto-generated method stub - try { - addAlgorithmsToHost(algoSet, hostname, uuid); - ***REMOVED*** catch (IOException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); - ***REMOVED*** this line will execute immediately, not waiting for your task to complete - System.out.println(uuid); - return uuid; - ***REMOVED*** - - - public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{ - - File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs"); + new Thread(new Runnable() { + @Override + public void run() { + ***REMOVED*** TODO Auto-generated method stub + try { + addAlgorithmsToVRE(algoSet, vre, uuid); + ***REMOVED*** catch (IOException e) { + ***REMOVED*** TODO Auto-generated catch block + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** + ***REMOVED***).start(); + ***REMOVED*** this line will execute immediately, not waiting for your task to + ***REMOVED*** complete + System.out.println(uuid); + return uuid; +***REMOVED*** + + public String addAlgorithmToHost(Algorithm algorithm, final String hostname) throws IOException { + ***REMOVED*** create a fake algorithm set + final AlgorithmSet algoSet = new AlgorithmSet(); + algoSet.setName("fake"); + algoSet.addAlgorithm(algorithm); + final String uuid = UUID.randomUUID().toString(); + + new Thread(new Runnable() { + @Override + public void run() { + ***REMOVED*** TODO Auto-generated method stub + try { + addAlgorithmsToHost(algoSet, hostname, uuid); + ***REMOVED*** catch (IOException e) { + ***REMOVED*** TODO Auto-generated catch block + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** + ***REMOVED***).start(); + ***REMOVED*** this line will execute immediately, not waiting for your task to + ***REMOVED*** complete + System.out.println(uuid); + return uuid; +***REMOVED*** + + public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException { + + File path = new File(System.getProperty("user.home") + File.separator + "/gcube/dataminer-pool-manager/work/" + + a + File.separator + "logs"); path.mkdirs(); - File n = new File(path + File.separator +a); - ***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress(); - + File n = new File(path + File.separator + a); + ***REMOVED*** String addr = InetAddress.getLocalHost().getHostAddress(); + return new File(n.getPath()).toURI().toURL(); - ***REMOVED*** - - - - public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException { - - ***REMOVED*** create the cluster (dataminers in the vre) - Cluster cluster = new Cluster(); - for(Host h:new ISClient().listDataminersInVRE()) { - cluster.addHost(h); - ***REMOVED*** - - ***REMOVED*** apply the changes - AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId(); - - ***REMOVED*** - - public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid) throws IOException { - - ***REMOVED*** create the cluster (dataminers in the vre) - Cluster cluster = new Cluster(); - cluster.addHost(new ISClient().getDataminer(hostname)); -***REMOVED*** for(Host h:new ISClient().listDataminersInVRE()) { -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** - - ***REMOVED*** apply the changes - AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId(); - - ***REMOVED*** - - - public Algorithm extractAlgorithm(String url) throws IOException { - return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** - - -@Override -public void getLogId(Algorithm algo, String vre) { - ***REMOVED*** TODO Auto-generated method stub - ***REMOVED*** + public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException { + + ***REMOVED*** create the cluster (dataminers in the vre) + Cluster cluster = new Cluster(); + for (Host h : new ISClient().listDataminersInVRE()) { + cluster.addHost(h); + ***REMOVED*** + + ***REMOVED*** apply the changes + AnsibleBridge a = new AnsibleBridge(); + return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid).getWorkerId(); -@Override -public String getLogById(String logId) throws IOException { - ***REMOVED*** TODO Auto-generated method stub - return null; ***REMOVED*** + public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid) throws IOException { + ***REMOVED*** create the cluster (dataminers in the vre) + Cluster cluster = new Cluster(); + cluster.addHost(new ISClient().getDataminer(hostname)); + ***REMOVED*** for(Host h:new ISClient().listDataminersInVRE()) { + ***REMOVED*** cluster.addHost(h); + ***REMOVED*** ***REMOVED*** + ***REMOVED*** apply the changes + AnsibleBridge a = new AnsibleBridge(); + return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid).getWorkerId(); - ***REMOVED*** + public Algorithm extractAlgorithm(String url) throws IOException { + return new AlgorithmPackageParser().parsePackage(url); +***REMOVED*** + @Override + public void getLogId(Algorithm algo, String vre) { + ***REMOVED*** TODO Auto-generated method stub + +***REMOVED*** + + @Override + public String getLogById(String logId) throws IOException { + ***REMOVED*** TODO Auto-generated method stub + return null; +***REMOVED*** + + ***REMOVED*** 2017 March 29 + + public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception { + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher); + String id = resource.id(); + LOGGER.debug("Trying to remove {***REMOVED*** with ID {***REMOVED*** from {***REMOVED***", resource.getClass().getSimpleName(), id, + ScopeProvider.instance.get()); + ***REMOVED*** scopedPublisher.remove(resource, scopes); + advancedScopedPublisher.forceRemove(resource); + LOGGER.debug("{***REMOVED*** with ID {***REMOVED*** removed successfully", resource.getClass().getSimpleName(), id); +***REMOVED*** + + public void publishScopedResource(GenericResource a, List scopes) + throws RegistryNotFoundException, Exception { + StringWriter stringWriter = new StringWriter(); + Resources.marshal(a, stringWriter); + + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + try { + System.out.println(scopes); + System.out.println(stringWriter); + scopedPublisher.create(a, scopes); + ***REMOVED*** catch (RegistryNotFoundException e) { + System.out.println(e); + throw e; + ***REMOVED*** +***REMOVED*** + + @Override + public void addAlgToIs(Algorithm algo) { + GenericResource a = new GenericResource(); + a.newProfile().name(algo.getName()).type("DataMinerAlgorithms").description(algo.getDescription()); + a.profile().newBody(this.getAlgoBody(algo)); + try { + publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() ***REMOVED***)); + ***REMOVED*** catch (Exception e) { + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + + private String getAlgoBody(Algorithm algo) { + return "" + algo.getId() + "" + "\n" + "" + algo.getCategory() + "" + "\n" + + "" + algo.getClazz() + "" + "\n" + "" + algo.getAlgorithmType() + + "" + "\n" + "" + algo.getSkipJava() + "" + "\n" + "" + + algo.getPackageURL() + "" + "\n" + "" + + algo.getDependencies().iterator().next().getName() + ""; +***REMOVED*** + + public void updateAlg(Algorithm algo) { + + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + + SimpleQuery query = queryFor(GenericResource.class); + query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource"); + DiscoveryClient client = clientFor(GenericResource.class); + List ds = client.submit(query); + if (ds.isEmpty()) { + return; + ***REMOVED*** + GenericResource a = ds.get(0); + a.profile().newBody(this.getAlgoBody(algo)); + try { + scopedPublisher.update(a); + ***REMOVED*** catch (RegistryNotFoundException e) { + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + + private Algorithm convertAlgo(GenericResource a) { + Algorithm out = new Algorithm(); + + out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); + out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); + out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); + out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); + out.setName(a.profile().name()); + out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); + out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); + out.setDescription(a.profile().description()); + + Set deps = new HashSet(); + for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) { + org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); + d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); + deps.add(d1); + ***REMOVED*** + out.setDependencies(deps); +***REMOVED*** +***REMOVED*** + + public Algorithm getAlgoById(String id) { + for (Algorithm aa : this.getAlgoFromIs()) { + if (aa.getId().equals(id)) { + return aa; + ***REMOVED*** + ***REMOVED*** + return null; +***REMOVED*** + + @Override + public Set getAlgoFromIs() { + ***REMOVED*** TODO Auto-generated method stub + + Set out = new HashSet(); + SimpleQuery query = queryFor(GenericResource.class); + query.addCondition("$resource/Profile/SecondaryType/text() eq 'DataMinerAlgorithms'").setResult("$resource"); + DiscoveryClient client = clientFor(GenericResource.class); + List ds = client.submit(query); + for (GenericResource a : ds) { + out.add(this.convertAlgo(a)); + ***REMOVED*** +***REMOVED*** +***REMOVED*** + +***REMOVED***