diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index 00cdc26..21a38b9 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -204,82 +204,82 @@ public class AnsibleBridge { ***REMOVED*** ***REMOVED*** - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { - - - return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); - ***REMOVED*** +***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { +***REMOVED*** +***REMOVED*** +***REMOVED*** return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); +***REMOVED*** ***REMOVED*** - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { - AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); - - - List algoRoles = new Vector<>(); - - ***REMOVED*** add algorithms and dependencies to the worker - for (Algorithm a : as.getAlgorithms()) { - for (Role r : this.generateRoles(a)) { - algoRoles.add(r); - worker.addRole(r); - ***REMOVED*** - ***REMOVED***to comment the for in case of just install algo - if(test){ - for (Dependency d : a.getDependencies()) { - for (Role r : this.generateRoles(d)) { - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - - ***REMOVED*** add static roles - for(Role r:this.getStaticRoleManager().getStaticRoles()) { - worker.addRole(r); - ***REMOVED*** - - ***REMOVED*** generate the inventory - Inventory inventory = new Inventory(); - for (Host h : cluster.getHosts()) { - AnsibleHost ah = new AnsibleHost(h.getName()); - inventory.addHost(ah, "universe"); - inventory.addHost(ah, "d4science"); - ***REMOVED*** - worker.setInventory(inventory); - - ***REMOVED*** generate the playbook - Playbook playbook = new Playbook(); - if(test){ - playbook.setRemote_user("root");***REMOVED*** - playbook.setRemote_user("gcube"); - playbook.applyTo("universe"); - for(Role r:algoRoles) { - ***REMOVED*** add only 'add' roles - if(!r.getName().endsWith("remove")) { - playbook.addRole(r.getName()); - ***REMOVED*** - ***REMOVED*** - - worker.setPlaybook(playbook); - - ***REMOVED*** execute and save log locally - ***REMOVED***PrintStream console = System.out; - File path = new File(worker.getWorkdir() + File.separator + "logs"); - path.mkdirs(); - File n = new File(path + File.separator + worker.getWorkerId()); - FileOutputStream fos = new FileOutputStream(n); - PrintStream ps = new PrintStream(fos); - - ***REMOVED***System.setErr(console); - - worker.apply(as,ps,test); - ***REMOVED***System.setOut(console); - ***REMOVED***worker.apply(); - System.out.println("Log stored to to " + n.getAbsolutePath()); - - ***REMOVED*** destroy the worker - worker.destroy(); - return worker; - ***REMOVED*** +***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { +***REMOVED*** AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); +***REMOVED*** +***REMOVED*** +***REMOVED*** List algoRoles = new Vector<>(); +***REMOVED*** +***REMOVED*** ***REMOVED*** add algorithms and dependencies to the worker +***REMOVED*** for (Algorithm a : as.getAlgorithms()) { +***REMOVED*** for (Role r : this.generateRoles(a)) { +***REMOVED*** algoRoles.add(r); +***REMOVED*** worker.addRole(r); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED***to comment the for in case of just install algo +***REMOVED*** if(test){ +***REMOVED*** for (Dependency d : a.getDependencies()) { +***REMOVED*** for (Role r : this.generateRoles(d)) { +***REMOVED*** worker.addRole(r); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** ***REMOVED*** add static roles +***REMOVED*** for(Role r:this.getStaticRoleManager().getStaticRoles()) { +***REMOVED*** worker.addRole(r); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** ***REMOVED*** generate the inventory +***REMOVED*** Inventory inventory = new Inventory(); +***REMOVED*** for (Host h : cluster.getHosts()) { +***REMOVED*** AnsibleHost ah = new AnsibleHost(h.getName()); +***REMOVED*** inventory.addHost(ah, "universe"); +***REMOVED*** inventory.addHost(ah, "d4science"); +***REMOVED*** ***REMOVED*** +***REMOVED*** worker.setInventory(inventory); +***REMOVED*** +***REMOVED*** ***REMOVED*** generate the playbook +***REMOVED*** Playbook playbook = new Playbook(); +***REMOVED*** if(test){ +***REMOVED*** playbook.setRemote_user("root");***REMOVED*** +***REMOVED*** playbook.setRemote_user("gcube"); +***REMOVED*** playbook.applyTo("universe"); +***REMOVED*** for(Role r:algoRoles) { +***REMOVED*** ***REMOVED*** add only 'add' roles +***REMOVED*** if(!r.getName().endsWith("remove")) { +***REMOVED*** playbook.addRole(r.getName()); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** worker.setPlaybook(playbook); +***REMOVED*** +***REMOVED*** ***REMOVED*** execute and save log locally +***REMOVED*** ***REMOVED***PrintStream console = System.out; +***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); +***REMOVED*** path.mkdirs(); +***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); +***REMOVED*** FileOutputStream fos = new FileOutputStream(n); +***REMOVED*** PrintStream ps = new PrintStream(fos); +***REMOVED*** +***REMOVED*** ***REMOVED***System.setErr(console); +***REMOVED*** +***REMOVED*** worker.apply(as,ps,test); +***REMOVED*** ***REMOVED***System.setOut(console); +***REMOVED*** ***REMOVED***worker.apply(); +***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); +***REMOVED*** +***REMOVED*** ***REMOVED*** destroy the worker +***REMOVED*** worker.destroy(); +***REMOVED*** return worker; +***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 4196712..6b51f72 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -172,70 +172,70 @@ public class DataminerPoolManager { return bufferScript; ***REMOVED*** - public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException { - ***REMOVED*** create a fake algorithm set - final AlgorithmSet algoSet = new AlgorithmSet(); - algoSet.setName("fake"); - algoSet.addAlgorithm(algorithm); - final String uuid = UUID.randomUUID().toString(); - - new Thread(new Runnable() { - @Override - public void run() { - ***REMOVED*** TODO Auto-generated method stub - try { - try { - addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test); - ***REMOVED*** catch (SVNException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** catch (IOException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** catch (InterruptedException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); - ***REMOVED*** this line will execute immediately, not waiting for task to - ***REMOVED*** complete - System.out.println(uuid); - return uuid; +***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException { +***REMOVED*** ***REMOVED*** create a fake algorithm set +***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet(); +***REMOVED*** algoSet.setName("fake"); +***REMOVED*** algoSet.addAlgorithm(algorithm); +***REMOVED*** final String uuid = UUID.randomUUID().toString(); ***REMOVED*** +***REMOVED*** new Thread(new Runnable() { +***REMOVED*** @Override +***REMOVED*** public void run() { +***REMOVED*** ***REMOVED*** TODO Auto-generated method stub +***REMOVED*** try { +***REMOVED*** try { +***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test); +***REMOVED*** ***REMOVED*** catch (SVNException e) { +***REMOVED*** ***REMOVED*** TODO Auto-generated catch block +***REMOVED*** e.printStackTrace(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** catch (IOException e) { +***REMOVED*** ***REMOVED*** TODO Auto-generated catch block +***REMOVED*** e.printStackTrace(); +***REMOVED*** ***REMOVED*** catch (InterruptedException e) { +***REMOVED*** ***REMOVED*** TODO Auto-generated catch block +***REMOVED*** e.printStackTrace(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED***).start(); +***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for task to +***REMOVED*** ***REMOVED*** complete +***REMOVED*** System.out.println(uuid); +***REMOVED*** return uuid; +***REMOVED******REMOVED*** - public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException { - ***REMOVED*** create a fake algorithm set - final AlgorithmSet algoSet = new AlgorithmSet(); - algoSet.setName("fake"); - algoSet.addAlgorithm(algorithm); - final String uuid = UUID.randomUUID().toString(); - - new Thread(new Runnable() { - @Override - public void run() { - ***REMOVED*** TODO Auto-generated method stub - try { - if(test){ - addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED*** - ***REMOVED*** catch (IOException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** catch (InterruptedException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** catch (SVNException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); - ***REMOVED*** this line will execute immediately, not waiting for your task to - ***REMOVED*** complete - System.out.println(uuid); - return uuid; +***REMOVED*** public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException { +***REMOVED*** ***REMOVED*** create a fake algorithm set +***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet(); +***REMOVED*** algoSet.setName("fake"); +***REMOVED*** algoSet.addAlgorithm(algorithm); +***REMOVED*** final String uuid = UUID.randomUUID().toString(); ***REMOVED*** +***REMOVED*** new Thread(new Runnable() { +***REMOVED*** @Override +***REMOVED*** public void run() { +***REMOVED*** ***REMOVED*** TODO Auto-generated method stub +***REMOVED*** try { +***REMOVED*** if(test){ +***REMOVED*** addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED*** +***REMOVED*** ***REMOVED*** catch (IOException e) { +***REMOVED*** ***REMOVED*** TODO Auto-generated catch block +***REMOVED*** e.printStackTrace(); +***REMOVED*** ***REMOVED*** catch (InterruptedException e) { +***REMOVED*** ***REMOVED*** TODO Auto-generated catch block +***REMOVED*** e.printStackTrace(); +***REMOVED*** ***REMOVED*** catch (SVNException e) { +***REMOVED*** ***REMOVED*** TODO Auto-generated catch block +***REMOVED*** e.printStackTrace(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED***).start(); +***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for your task to +***REMOVED*** ***REMOVED*** complete +***REMOVED*** System.out.println(uuid); +***REMOVED*** return uuid; +***REMOVED******REMOVED*** public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException { @@ -249,52 +249,52 @@ public class DataminerPoolManager { return new File(n.getPath()).toURI().toURL(); ***REMOVED*** - public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { - - ***REMOVED*** create the cluster (dataminers in the vre) - Cluster cluster = new Cluster(); - for (Host h : new HAProxy().listDataMinersByCluster()) { - ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) { - cluster.addHost(h); - ***REMOVED*** - - ***REMOVED*** apply the changes - AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId(); - +***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { ***REMOVED*** - - public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) - throws IOException, InterruptedException, SVNException { - - ***REMOVED*** create the cluster (dataminers in the vre) - Cluster cluster = new Cluster(); - for (Host h : new HAProxy().listDataMinersByCluster()) { - if (h.getName().equals(hostname)) { - cluster.addHost(h); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** if(ISClient.getHProxy().equals(hostname)){ - ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname)); - ***REMOVED*** ***REMOVED*** - ***REMOVED*** apply the changes - AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); - +***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre) +***REMOVED*** Cluster cluster = new Cluster(); +***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) { +***REMOVED*** ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) { +***REMOVED*** cluster.addHost(h); +***REMOVED*** ***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** apply the changes +***REMOVED*** AnsibleBridge a = new AnsibleBridge(); +***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId(); +***REMOVED*** +***REMOVED******REMOVED*** + +***REMOVED*** public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) +***REMOVED*** throws IOException, InterruptedException, SVNException { +***REMOVED*** +***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre) +***REMOVED*** Cluster cluster = new Cluster(); +***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) { +***REMOVED*** if (h.getName().equals(hostname)) { +***REMOVED*** cluster.addHost(h); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** if(ISClient.getHProxy().equals(hostname)){ +***REMOVED*** ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname)); +***REMOVED*** ***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** apply the changes +***REMOVED*** AnsibleBridge a = new AnsibleBridge(); +***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); +***REMOVED*** +***REMOVED******REMOVED*** - - public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) - throws IOException, InterruptedException, SVNException { - Cluster cluster = new Cluster(); - Host h = new Host(); - h.setName(hostname); - cluster.addHost(h); - - AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); - +***REMOVED*** +***REMOVED*** public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) +***REMOVED*** throws IOException, InterruptedException, SVNException { +***REMOVED*** Cluster cluster = new Cluster(); +***REMOVED*** Host h = new Host(); +***REMOVED*** h.setName(hostname); +***REMOVED*** cluster.addHost(h); +***REMOVED*** +***REMOVED*** AnsibleBridge a = new AnsibleBridge(); +***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); ***REMOVED*** +***REMOVED******REMOVED*** ***REMOVED*** 2017 March 29 diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java index 0e1f345..3299c09 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java @@ -23,29 +23,22 @@ public class AlgorithmBuilder { if(category != null){ algo.setCategory(category); ***REMOVED*** - - ***REMOVED***FIXME: do the same done for category - if (algo.getAlgorithmType() == null) { - algo.setAlgorithmType(algorithmType); - ***REMOVED*** else - algo.setAlgorithmType(algo.getCategory()); - - if (algo.getSkipJava() == null) { - algo.setSkipJava(skipJava); - ***REMOVED*** else - algo.setSkipJava(algo.getSkipJava()); - - if (algo.getName() == null) { - algo.setName(name); - ***REMOVED*** else - algo.setName(algo.getName()); - - if (algo.getDescription() == null) { - algo.setDescription(description); - ; - ***REMOVED*** else - algo.setDescription(algo.getDescription()); - + if(algorithmType != null){ + algo.setAlgorithmType(algorithmType); + ***REMOVED*** + if(skipJava != null){ + algo.setSkipJava(skipJava); + ***REMOVED*** + if(skipJava != null){ + algo.setSkipJava(skipJava); + ***REMOVED*** + if(name != null){ + algo.setName(name); + ***REMOVED*** + if(description != null){ + algo.setDescription(description); + ***REMOVED*** + return algo; ***REMOVED*** diff --git a/src/test/java/DataminerPoolManagerTest.java b/src/test/java/DataminerPoolManagerTest.java index 4e1b981..ac7730f 100644 --- a/src/test/java/DataminerPoolManagerTest.java +++ b/src/test/java/DataminerPoolManagerTest.java @@ -220,7 +220,7 @@ public class DataminerPoolManagerTest { ensemble.addDependency(d); algorithms.addAlgorithm(ensemble); - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); + ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); ***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java index 2477675..180b1e5 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java @@ -220,7 +220,7 @@ public class DataminerPoolManagerTest { ensemble.addDependency(d); algorithms.addAlgorithm(ensemble); - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); + ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); ***REMOVED***