From 3d6a1273ffea31f4057f6f6a326fcb4f40222233 Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Tue, 9 May 2017 15:10:03 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148412 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../poolmanager/ansible/AnsibleWorker.java | 54 ++- .../ansiblebridge/AnsibleBridge.java | 15 +- .../poolmanager/clients/ISClient.java | 1 + .../poolmanager/rest/PoolManager.java | 7 +- .../poolmanager/rest/RestPoolManager.java | 98 ++-- .../service/DataminerPoolManager.java | 431 ++---------------- 6 files changed, 154 insertions(+), 452 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index 5fb8ef2..2b6660a 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -24,6 +24,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSeriali import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; +import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; @@ -118,7 +119,7 @@ public class AnsibleWorker { - public void apply(AlgorithmSet as, PrintStream ps, boolean updateSVN) + public void apply(AlgorithmSet as, PrintStream ps, boolean test) throws IOException, InterruptedException, SVNException { ***REMOVED*** TODO execute the playbook and return output System.out.println(this.getWorkdir()); @@ -128,43 +129,70 @@ public class AnsibleWorker { inheritIO(p.getInputStream(), ps); inheritIO(p.getErrorStream(), ps); - - if (updateSVN) { - int exitValue = p.waitFor(); - if (exitValue == 0) { - + + + int exitValue = p.waitFor(); + if (exitValue == 0) { + if (test) { for (Algorithm algo : as.getAlgorithms()) { - + for (Dependency d : algo.getDependencies()) { if (d.getType().equals("os")) { List ls = new LinkedList(); ls.add(d.getName()); - this.updateSVN("r_deb_pkgs.txt", ls); + this.updateSVN("test_r_deb_pkgs.txt", ls); ***REMOVED*** if (d.getType().equals("cran")) { List ls = new LinkedList(); ls.add(d.getName()); - this.updateSVN("r_cran_pkgs.txt", ls); + this.updateSVN("test_r_cran_pkgs.txt", ls); ***REMOVED*** if (d.getType().equals("github")) { List ls = new LinkedList(); ls.add(d.getName()); - this.updateSVN("r_github_pkgs.txt", ls); + this.updateSVN("test_r_github_pkgs.txt", ls); ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED*** + else if(!test){ + for (Algorithm algo : as.getAlgorithms()) { + + for (Dependency d : algo.getDependencies()) { + if (d.getType().equals("os")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN("r_deb_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("cran")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN("r_cran_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("github")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN("r_github_pkgs.txt", ls); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + + for (Algorithm algo : as.getAlgorithms()) { + DataminerPoolManager a = new DataminerPoolManager(); + a.addAlgToIs(algo); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** catch (IOException e) { e.printStackTrace(); ***REMOVED*** - ***REMOVED*** System.out.println("TODO: execute: ansible-playbook -v -i " + - ***REMOVED*** this.getInventoryFile().getName() + " " + - ***REMOVED*** this.getPlaybookFile().getName()); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index 5319d37..b5a4096 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -158,13 +158,13 @@ public class AnsibleBridge { ***REMOVED*** ***REMOVED*** - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, boolean updateSVN) throws IOException, InterruptedException, SVNException { + public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { - return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),updateSVN); + return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); ***REMOVED*** - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException { + public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); @@ -176,11 +176,14 @@ public class AnsibleBridge { algoRoles.add(r); worker.addRole(r); ***REMOVED*** + ***REMOVED***to comment the for in case of just install algo + if(test){ for (Dependency d : a.getDependencies()) { for (Role r : this.generateRoles(d)) { worker.addRole(r); ***REMOVED*** ***REMOVED*** + ***REMOVED*** ***REMOVED*** ***REMOVED*** add static roles @@ -199,7 +202,9 @@ public class AnsibleBridge { ***REMOVED*** generate the playbook Playbook playbook = new Playbook(); - playbook.setRemote_user("root"); + if(test){ + playbook.setRemote_user("root");***REMOVED*** + playbook.setRemote_user("gcube"); playbook.applyTo("universe"); for(Role r:algoRoles) { ***REMOVED*** add only 'add' roles @@ -220,7 +225,7 @@ public class AnsibleBridge { ***REMOVED***System.setErr(console); - worker.apply(as,ps,updateSVN); + worker.apply(as,ps,test); ***REMOVED***System.setOut(console); ***REMOVED***worker.apply(); System.out.println("Log stored to to " + n.getAbsolutePath()); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index 3707cbb..5acd531 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -72,6 +72,7 @@ public class ISClient { ***REMOVED*** return h; ***REMOVED*** + ***REMOVED*** return the HProxy hostname in the VRE public static String getHProxy() { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index d0fa7a1..60a2c6b 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -35,12 +35,13 @@ import java.net.UnknownHostException; import java.util.Set; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.tmatesoft.svn.core.SVNException; public interface PoolManager { - String addAlgorithmToVRE(Algorithm algo, String vre, boolean svn ) throws IOException, InterruptedException; - String addAlgorithmToHost(Algorithm algo, String host, boolean svn) throws IOException, InterruptedException; + String addAlgorithmToVRE(Algorithm algo, String vre, boolean test ) throws IOException, InterruptedException; + String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException; Algorithm extractAlgorithm(String url) throws IOException; @@ -57,5 +58,5 @@ public interface PoolManager { Set getAlgoFromIs(); List updateSVN(String file, List ldep) throws SVNException, IOException; - + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index df78dab..fbd63a2 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -162,8 +162,10 @@ import javax.ws.rs.QueryParam; import org.apache.commons.lang.math.RandomUtils; ***REMOVED*** +***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; @@ -171,6 +173,8 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySel ***REMOVED*** import org.tmatesoft.svn.core.SVNException; +import ch.qos.logback.core.subst.Token; + @Path("/") public class RestPoolManager implements PoolManager { @@ -201,37 +205,46 @@ public class RestPoolManager implements PoolManager { if (publish) { service.addAlgToIs(algo); ***REMOVED*** - return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),updateSVN); + return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),/*updateSVN,*/false); ***REMOVED*** - - - + + ***REMOVED***Test Phase + ***REMOVED***The DMPM contacts the Staging Dataminer and installs the algorithm and the dependencies @GET - @Path("/hosts/add") + @Path("/algo/add") @Produces("text/plain") - public String addAlgorithmToHost( - @QueryParam("algorithm") String algorithm, - @QueryParam("hostname") String hostname, + public String addAlgorithmToStagingHost( + @QueryParam("algorithm") String algorithm, + @DefaultValue("insert hostname here") @QueryParam("hostname") String hostname, @QueryParam("name") String name, @QueryParam("description") String description, @QueryParam("category") String category, @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType, @DefaultValue("N") @QueryParam("skipJava") String skipJava, - @DefaultValue("false") @QueryParam("publish") boolean publish, - @DefaultValue("false") @QueryParam("updateSVN") boolean updateSVN) + @DefaultValue("prodtoken") @QueryParam("prodtoken") String prodtoken, + @DefaultValue("true") @QueryParam("test") boolean test) throws IOException, InterruptedException, SVNException { - Algorithm algo = this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, - skipJava); - ***REMOVED*** publish algo - if (publish) { - service.addAlgToIs(algo); + String id=""; + Algorithm algo = this.getAlgorithm(algorithm, ScopeProvider.instance.get(), hostname, name, description, category, algorithmType,skipJava); + + ***REMOVED***test phase + if(test){ + service.addAlgorithmToHost(algo, hostname,true); + id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),true); ***REMOVED*** - return service.addAlgorithmToHost(algo, hostname,updateSVN); + ***REMOVED***release phase + if(!test){ + SecurityTokenProvider.instance.set(prodtoken); + id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),false); + ***REMOVED*** + + return id; + ***REMOVED*** - + + private Algorithm getAlgorithm(String algorithm, String vre, String hostname, String name, String description, String category, String algorithmType, String skipJava) throws IOException, InterruptedException { - LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + ScopeProvider.instance.get()); Algorithm algo = service.extractAlgorithm(algorithm); if (algo.getCategory() == null) { @@ -283,8 +296,10 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** ProxySelector.setDefault(new ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - + ***REMOVED***ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); + + ***REMOVED*** PoolManager aa = new DataminerPoolManager(); ***REMOVED*** System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3")); ***REMOVED*** System.out.println(aa.getAlgoFromIs()); @@ -306,9 +321,17 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** "dataminer1-pre.d4science.org", ***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", ***REMOVED*** "N",false, false); - - + RestPoolManager b = new RestPoolManager(); + b.addAlgorithmToStagingHost( + "http:***REMOVED***data.d4science.org/TVc0TW9Ud1FjYlppK0NHd2pvU0owNmRFWHE4OW4xSGNHbWJQNStIS0N6Yz0", + "dataminer1-devnext.d4science.org", + null, + null, + "N", + "transducerers", + false, + false ***REMOVED*** PoolManager aa = new DataminerPoolManager(); ***REMOVED*** List ls = new LinkedList(); ***REMOVED*** String afa = "test"; @@ -357,21 +380,32 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** - @Override - public String addAlgorithmToVRE(Algorithm algo, String vre, boolean svn) throws IOException, InterruptedException { - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** - - - - @Override - public String addAlgorithmToHost(Algorithm algo, String host, boolean svn) + public String addAlgorithmToHost(Algorithm algo, String host, boolean test) throws IOException, InterruptedException { ***REMOVED*** TODO Auto-generated method stub return null; ***REMOVED*** + + + @Override + public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test) + throws IOException, InterruptedException { + ***REMOVED*** TODO Auto-generated method stub + return null; +***REMOVED*** + + + + + + + + + + + + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index e01d48f..c5d3e58 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -1,265 +1,3 @@ -***REMOVED***package org.gcube.dataanalysis.dataminer.poolmanager.service; -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED***import java.io.File; -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED***import java.net.MalformedURLException; -***REMOVED******REMOVED*** -***REMOVED***import java.net.URLConnection; -***REMOVED***import java.net.UnknownHostException; -***REMOVED***import java.util.UUID; -***REMOVED*** -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager; -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED***public class DataminerPoolManager implements PoolManager { -***REMOVED*** -***REMOVED*** -***REMOVED*** private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); -***REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** static Collection algorithms; -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** static Collection sets; -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** static { -***REMOVED******REMOVED*** algorithms = new Vector<>(); -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** public DataminerPoolManager() { -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** /** -***REMOVED******REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected -***REMOVED******REMOVED*** * on the pool. -***REMOVED******REMOVED*** */ -***REMOVED******REMOVED*** public void publishAlgorithm(Algorithm algorithm) { -***REMOVED******REMOVED*** algorithms.add(algorithm); -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** /** -***REMOVED******REMOVED*** * Re-deploy the given algorithm wherever it's installed -***REMOVED******REMOVED*** * -***REMOVED******REMOVED*** * @param algorithm -***REMOVED******REMOVED*** */ -***REMOVED******REMOVED*** /* -***REMOVED******REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement this ***REMOVED*** -***REMOVED******REMOVED*** */ -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** /** -***REMOVED******REMOVED*** * Add the give algorithm to the given set -***REMOVED******REMOVED*** * -***REMOVED******REMOVED*** * @param algorithmId -***REMOVED******REMOVED*** * @param setId -***REMOVED******REMOVED*** */ -***REMOVED******REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) { -***REMOVED******REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); -***REMOVED******REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName); -***REMOVED******REMOVED*** if (set != null && algorithm != null) { -***REMOVED******REMOVED*** set.addAlgorithm(algorithm); -***REMOVED******REMOVED*** this.updateClusters(); -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** /** -***REMOVED******REMOVED*** * Apply the given set of algorithms to the given cluster -***REMOVED******REMOVED*** * -***REMOVED******REMOVED*** * @param setId -***REMOVED******REMOVED*** * @param clusterId -***REMOVED******REMOVED*** */ -***REMOVED******REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) { -***REMOVED******REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); -***REMOVED******REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName); -***REMOVED******REMOVED*** if (set != null && cluster != null) { -***REMOVED******REMOVED*** cluster.addAlgorithmSet(set); -***REMOVED******REMOVED*** this.updateClusters(); -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** private AlgorithmSet getAlgorithmSet(String name) { -***REMOVED******REMOVED*** for (AlgorithmSet set : sets) { -***REMOVED******REMOVED*** if (name.equals(set.getName())) { -***REMOVED******REMOVED*** return set; -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** return null; -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** private Algorithm getAlgorithm(String name) { -***REMOVED******REMOVED*** for (Algorithm a : algorithms) { -***REMOVED******REMOVED*** if (name.equals(a.getName())) { -***REMOVED******REMOVED*** return a; -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** return null; -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) { -***REMOVED******REMOVED*** new Thread() { -***REMOVED******REMOVED*** public void run() { -***REMOVED******REMOVED*** while (true) { -***REMOVED******REMOVED*** try { -***REMOVED******REMOVED*** addAlgorithmToVRE(algorithm, vre); -***REMOVED******REMOVED*** ***REMOVED*** catch (Exception e) { -***REMOVED******REMOVED*** ***REMOVED***log here -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** ***REMOVED***.start(); -***REMOVED******REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** public String getLogId(){ -***REMOVED******REMOVED*** PrintStream console = System.out; -***REMOVED******REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); -***REMOVED******REMOVED*** path.mkdirs(); -***REMOVED******REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); -***REMOVED******REMOVED*** FileOutputStream fos = new FileOutputStream(n); -***REMOVED******REMOVED*** PrintStream ps = new PrintStream(fos); -***REMOVED******REMOVED*** System.setOut(ps); -***REMOVED******REMOVED*** worker.apply(); -***REMOVED******REMOVED*** System.setOut(console); -***REMOVED******REMOVED*** worker.apply(); -***REMOVED******REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); -***REMOVED******REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** public String getLogById(String id) throws IOException { -***REMOVED******REMOVED*** String strLine = null; -***REMOVED******REMOVED*** try{ -***REMOVED******REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id); -***REMOVED******REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); -***REMOVED******REMOVED*** /* read log line by line */ -***REMOVED******REMOVED*** while ((strLine = br.readLine()) != null) { -***REMOVED******REMOVED*** /* parse strLine to obtain what you want */ -***REMOVED******REMOVED*** System.out.println (strLine); -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** br.close(); -***REMOVED******REMOVED*** ***REMOVED*** catch (Exception e) { -***REMOVED******REMOVED*** System.err.println("Error: " + e.getMessage()); -***REMOVED******REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** return strLine; -***REMOVED******REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** public String getScriptFromURL(URL url) throws IOException { -***REMOVED*** if (url == null) { -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** -***REMOVED*** URLConnection yc = url.openConnection(); -***REMOVED*** BufferedReader input = new BufferedReader(new InputStreamReader( -***REMOVED*** yc.getInputStream())); -***REMOVED*** String line; -***REMOVED*** StringBuffer buffer = new StringBuffer(); -***REMOVED*** while ((line = input.readLine()) != null) { -***REMOVED*** buffer.append(line + "\n"); -***REMOVED*** ***REMOVED*** -***REMOVED*** String bufferScript = buffer.substring(0, buffer.length()); -***REMOVED*** input.close(); -***REMOVED*** return bufferScript; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Publish the given algorithm in the given VRE -***REMOVED*** * -***REMOVED*** * @param algorithmName -***REMOVED*** * @param vre -***REMOVED*** * -***REMOVED*** */ -***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException { -***REMOVED*** ***REMOVED*** create a fake algorithm set -***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet(); -***REMOVED*** algoSet.setName("fake"); -***REMOVED*** algoSet.addAlgorithm(algorithm); -***REMOVED*** final String uuid = UUID.randomUUID().toString(); -***REMOVED*** -***REMOVED*** new Thread(new Runnable() { -***REMOVED*** @Override -***REMOVED*** public void run() { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** try { -***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid); -***REMOVED*** ***REMOVED*** catch (IOException e) { -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED***).start(); -***REMOVED*** ***REMOVED***this line will execute immediately, not waiting for your task to complete -***REMOVED*** System.out.println(uuid); -***REMOVED*** return uuid; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{ -***REMOVED*** -***REMOVED*** File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs"); -***REMOVED*** path.mkdirs(); -***REMOVED*** File n = new File(path + File.separator +a); -***REMOVED*** ***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress(); -***REMOVED*** -***REMOVED*** return new File(n.getPath()).toURI().toURL(); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException { -***REMOVED*** -***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre) -***REMOVED*** Cluster cluster = new Cluster(); -***REMOVED*** for(Host h:new ISClient().listDataminersInVRE()) { -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** ***REMOVED*** apply the changes -***REMOVED*** AnsibleBridge a = new AnsibleBridge(); -***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId(); -***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public Algorithm extractAlgorithm(String url) throws IOException { -***REMOVED*** return new AlgorithmPackageParser().parsePackage(url); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED***@Override -***REMOVED***public void getLogId(Algorithm algo, String vre) { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED***@Override -***REMOVED***public String getLogById(String logId) throws IOException { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** return null; -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** package org.gcube.dataanalysis.dataminer.poolmanager.service; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; @@ -325,134 +63,7 @@ public class DataminerPoolManager implements PoolManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); - ***REMOVED*** static Collection algorithms; - ***REMOVED*** - ***REMOVED*** static Collection sets; - ***REMOVED*** - ***REMOVED*** static { - ***REMOVED*** algorithms = new Vector<>(); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** public DataminerPoolManager() { - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** /** - ***REMOVED*** * Add a new algorithm to the set of known ones. No further action is - ***REMOVED*** expected - ***REMOVED*** * on the pool. - ***REMOVED*** */ - ***REMOVED*** public void publishAlgorithm(Algorithm algorithm) { - ***REMOVED*** algorithms.add(algorithm); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** /** - ***REMOVED*** * Re-deploy the given algorithm wherever it's installed - ***REMOVED*** * - ***REMOVED*** * @param algorithm - ***REMOVED*** */ - ***REMOVED*** /* - ***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement - ***REMOVED*** this ***REMOVED*** - ***REMOVED*** */ - ***REMOVED*** - ***REMOVED*** /** - ***REMOVED*** * Add the give algorithm to the given set - ***REMOVED*** * - ***REMOVED*** * @param algorithmId - ***REMOVED*** * @param setId - ***REMOVED*** */ - ***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) { - ***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); - ***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName); - ***REMOVED*** if (set != null && algorithm != null) { - ***REMOVED*** set.addAlgorithm(algorithm); - ***REMOVED*** this.updateClusters(); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** /** - ***REMOVED*** * Apply the given set of algorithms to the given cluster - ***REMOVED*** * - ***REMOVED*** * @param setId - ***REMOVED*** * @param clusterId - ***REMOVED*** */ - ***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String - ***REMOVED*** clusterName) { - ***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); - ***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName); - ***REMOVED*** if (set != null && cluster != null) { - ***REMOVED*** cluster.addAlgorithmSet(set); - ***REMOVED*** this.updateClusters(); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) { - ***REMOVED*** for (AlgorithmSet set : sets) { - ***REMOVED*** if (name.equals(set.getName())) { - ***REMOVED*** return set; - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** return null; - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** private Algorithm getAlgorithm(String name) { - ***REMOVED*** for (Algorithm a : algorithms) { - ***REMOVED*** if (name.equals(a.getName())) { - ***REMOVED*** return a; - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** return null; - ***REMOVED*** ***REMOVED*** - - ***REMOVED*** - ***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) { - ***REMOVED*** new Thread() { - ***REMOVED*** public void run() { - ***REMOVED*** while (true) { - ***REMOVED*** try { - ***REMOVED*** addAlgorithmToVRE(algorithm, vre); - ***REMOVED*** ***REMOVED*** catch (Exception e) { - ***REMOVED*** ***REMOVED***log here - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED***.start(); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - - ***REMOVED*** public String getLogId(){ - ***REMOVED*** PrintStream console = System.out; - ***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); - ***REMOVED*** path.mkdirs(); - ***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); - ***REMOVED*** FileOutputStream fos = new FileOutputStream(n); - ***REMOVED*** PrintStream ps = new PrintStream(fos); - ***REMOVED*** System.setOut(ps); - ***REMOVED*** worker.apply(); - ***REMOVED*** System.setOut(console); - ***REMOVED*** worker.apply(); - ***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); - ***REMOVED*** ***REMOVED*** - - ***REMOVED*** public String getLogById(String id) throws IOException { - ***REMOVED*** String strLine = null; - ***REMOVED*** try{ - ***REMOVED*** FileInputStream fstream = new - ***REMOVED*** FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id); - ***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); - ***REMOVED*** /* read log line by line */ - ***REMOVED*** while ((strLine = br.readLine()) != null) { - ***REMOVED*** /* parse strLine to obtain what you want */ - ***REMOVED*** System.out.println (strLine); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** br.close(); - ***REMOVED*** ***REMOVED*** catch (Exception e) { - ***REMOVED*** System.err.println("Error: " + e.getMessage()); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** return strLine; - ***REMOVED*** ***REMOVED*** - + public String getScriptFromURL(URL url) throws IOException { if (url == null) { return null; @@ -476,7 +87,7 @@ public class DataminerPoolManager implements PoolManager { * @param vre * */ - public String addAlgorithmToVRE(Algorithm algorithm, final String vre, final boolean updateSVN) throws IOException { + public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException { ***REMOVED*** create a fake algorithm set final AlgorithmSet algoSet = new AlgorithmSet(); algoSet.setName("fake"); @@ -489,7 +100,7 @@ public class DataminerPoolManager implements PoolManager { ***REMOVED*** TODO Auto-generated method stub try { try { - addAlgorithmsToVRE(algoSet, vre, uuid, updateSVN); + addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test); ***REMOVED*** catch (SVNException e) { ***REMOVED*** TODO Auto-generated catch block e.printStackTrace(); @@ -503,13 +114,13 @@ public class DataminerPoolManager implements PoolManager { ***REMOVED*** ***REMOVED*** ***REMOVED***).start(); - ***REMOVED*** this line will execute immediately, not waiting for your task to + ***REMOVED*** this line will execute immediately, not waiting for task to ***REMOVED*** complete System.out.println(uuid); return uuid; ***REMOVED*** - public String addAlgorithmToHost(Algorithm algorithm, final String hostname, final boolean updateSVN) throws IOException { + public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException { ***REMOVED*** create a fake algorithm set final AlgorithmSet algoSet = new AlgorithmSet(); algoSet.setName("fake"); @@ -521,7 +132,8 @@ public class DataminerPoolManager implements PoolManager { public void run() { ***REMOVED*** TODO Auto-generated method stub try { - addAlgorithmsToHost(algoSet, hostname, uuid, updateSVN); + if(test){ + addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED*** ***REMOVED*** catch (IOException e) { ***REMOVED*** TODO Auto-generated catch block e.printStackTrace(); @@ -551,7 +163,7 @@ public class DataminerPoolManager implements PoolManager { return new File(n.getPath()).toURI().toURL(); ***REMOVED*** - public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException { + public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { ***REMOVED*** create the cluster (dataminers in the vre) Cluster cluster = new Cluster(); @@ -562,11 +174,11 @@ public class DataminerPoolManager implements PoolManager { ***REMOVED*** apply the changes AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, updateSVN).getWorkerId(); + return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId(); ***REMOVED*** - public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, boolean updateSVN) + public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) throws IOException, InterruptedException, SVNException { ***REMOVED*** create the cluster (dataminers in the vre) @@ -581,11 +193,23 @@ public class DataminerPoolManager implements PoolManager { ***REMOVED*** ***REMOVED*** ***REMOVED*** apply the changes AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, updateSVN).getWorkerId(); + return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); ***REMOVED*** + + public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) + throws IOException, InterruptedException, SVNException { + Cluster cluster = new Cluster(); + Host h = new Host(); + h.setName(hostname); + cluster.addHost(h); + + AnsibleBridge a = new AnsibleBridge(); + return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); +***REMOVED*** + public Algorithm extractAlgorithm(String url) throws IOException { return new AlgorithmPackageParser().parsePackage(url); @@ -726,4 +350,13 @@ public class DataminerPoolManager implements PoolManager { return null; ***REMOVED*** + + + + + + + + + ***REMOVED***