From 87218738ec40a6d2bac44836352f6a8d5539618e Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Tue, 21 Feb 2017 13:42:26 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@144015 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../poolmanager/ansible/AnsibleWorker.java | 33 +++++--- .../poolmanager/ansible/model/Playbook.java | 18 +++++ .../ansiblebridge/AnsibleBridge.java | 1 + .../ansiblebridge/AnsibleSerializeHelper.java | 2 +- .../poolmanager/clients/ISClient.java | 17 ++-- .../poolmanager/rest/RestPoolManager.java | 2 +- .../service/DataminerPoolManager.java | 5 +- .../DataminerPoolManagerTest.java | 81 ++++++++++--------- 8 files changed, 97 insertions(+), 62 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index 45b5d88..8be9d4c 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -3,8 +3,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible; ***REMOVED*** import java.io.File; ***REMOVED*** +import java.io.InputStream; ***REMOVED*** -import java.io.OutputStream; +import java.io.PrintStream; +import java.util.Scanner; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; @@ -65,7 +67,7 @@ public class AnsibleWorker { public void removeWorkStructure() { ***REMOVED*** remove the working dir - ***REMOVED***this.getWorkdir().delete(); + this.getWorkdir().delete(); ***REMOVED*** public File getPlaybookFile() { @@ -96,21 +98,32 @@ public class AnsibleWorker { public void apply() throws IOException { ***REMOVED*** TODO execute the playbook and return output + System.out.println(this.getWorkdir()); try { - System.out.println("ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName()); - Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName()); - BufferedReader in = new BufferedReader( - new InputStreamReader(p.getInputStream())); - String line = null; - while ((line = in.readLine()) != null) { - System.out.println(line); - ***REMOVED*** + Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath()); + + inheritIO(p.getInputStream(), System.out); + inheritIO(p.getErrorStream(), System.err); + ***REMOVED*** catch (IOException e) { e.printStackTrace(); ***REMOVED*** ***REMOVED***System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName()); ***REMOVED*** + + + private static void inheritIO(final InputStream src, final PrintStream dest) { + new Thread(new Runnable() { + public void run() { + Scanner sc = new Scanner(src); + while (sc.hasNextLine()) { + dest.println(sc.nextLine()); + ***REMOVED*** + ***REMOVED*** + ***REMOVED***).start(); +***REMOVED*** + /** * Destroy the worker: * - remove the working dir diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java index eeec135..d25ce80 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java @@ -9,6 +9,8 @@ public class Playbook { private List roles; + private String remote_user; + public Playbook() { this.roles = new Vector<>(); ***REMOVED*** @@ -28,5 +30,21 @@ public class Playbook { public List getRoles() { return new Vector<>(roles); ***REMOVED*** + +public String getRemote_user() { + return remote_user; +***REMOVED*** + +public void setRemote_user(String remote_user) { + this.remote_user = remote_user; +***REMOVED*** + +public void setHostGroupName(String hostGroupName) { + this.hostGroupName = hostGroupName; +***REMOVED*** + +public void setRoles(List roles) { + this.roles = roles; +***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index 7be99c0..a5f61b6 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -184,6 +184,7 @@ public class AnsibleBridge { ***REMOVED*** generate the playbook Playbook playbook = new Playbook(); + playbook.setRemote_user("root"); playbook.applyTo("universe"); for(Role r:algoRoles) { ***REMOVED*** add only 'add' roles diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java index 0cda968..48d9197 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java @@ -31,7 +31,7 @@ public class AnsibleSerializeHelper { public static void serialize(Playbook playbook, File playbookFile) throws IOException { String out = "- hosts: " + playbook.getHostGroupName() + "\n"; - out += " remote_user: dpm\n"; + out += " remote_user: "+playbook.getRemote_user()+"\n"; out+=" roles:\n"; for(String r:playbook.getRoles()) { out+=" - " + r+"\n"; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index 874c97c..627b988 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -37,9 +37,12 @@ public class ISClient { SimpleQuery query = queryFor(ServiceEndpoint.class); - query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'") - .addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); + ***REMOVED***old version + ***REMOVED***query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'") + ***REMOVED***.addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); + query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'"); + DiscoveryClient client = clientFor(ServiceEndpoint.class); List resources = client.submit(query); @@ -54,9 +57,9 @@ public class ISClient { ***REMOVED*** ***REMOVED*** -***REMOVED*** public static void main(String[] args) { -***REMOVED*** ISClient a = new ISClient(); -***REMOVED*** ScopeProvider.instance.set("/gcube/devNext/NextNext"); -***REMOVED*** System.out.println(a.listDataminersInVRE()); -***REMOVED******REMOVED*** + public static void main(String[] args) { + ISClient a = new ISClient(); + ScopeProvider.instance.set("/gcube/devNext/NextNext"); + System.out.println(a.listDataminersInVRE()); +***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index 0a8320b..1ecbe3b 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -59,7 +59,7 @@ public class RestPoolManager implements PoolManager { public static void main(String[] args) throws IOException, InterruptedException { RestPoolManager a = new RestPoolManager(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); - a.addAlgorithmToVRE("http:***REMOVED***data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0", "/gcube/devNext/NextNext"); + a.addAlgorithmToVRE("http:***REMOVED***data.d4science.org/eDF4dWtTc000dEpSLzcybU4zRmJoTmFwUnhZeGZBN1dHbWJQNStIS0N6Yz0", "/gcube/devNext/NextNext"); ***REMOVED***System.out.println(a.getLogById("dadcb059-69e5-48c3-aa58-3b290ae0419d")); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index b426609..4fb39e7 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -193,7 +193,7 @@ public class DataminerPoolManager implements PoolManager { algoSet.addAlgorithm(algorithm); final String uuid = UUID.randomUUID().toString(); - Runnable r = new Runnable() { + new Thread(new Runnable() { @Override public void run() { ***REMOVED*** TODO Auto-generated method stub @@ -204,8 +204,7 @@ public class DataminerPoolManager implements PoolManager { e.printStackTrace(); ***REMOVED*** ***REMOVED*** -***REMOVED***; - new Thread(r).start(); +***REMOVED***).start(); ***REMOVED***this line will execute immediately, not waiting for your task to complete System.out.println(uuid); return uuid; diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java index 37a2be5..6259dbc 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java @@ -6,6 +6,7 @@ import java.net.ProxySelector; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; +import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; public class DataminerPoolManagerTest { @@ -160,58 +161,58 @@ public class DataminerPoolManagerTest { ScopeProvider.instance.set("/gcube/devNext/NextNext"); ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - - ***REMOVED*** create the algorithm (download it, etc etc) - Algorithm algorithm = new Algorithm(); - algorithm.setName("ichtyop"); - algorithm.setClazz("org.gcube..."); - algorithm.setDescription("some description"); - - Dependency d = new Dependency(); - d.setName("libpng"); - d.setType("os"); - algorithm.addDependency(d); - - d = new Dependency(); - d.setName("some-r-package"); - d.setType("cran"); - algorithm.addDependency(d); - - d = new Dependency(); - d.setName("some-other-r-package"); - d.setType("cran"); - algorithm.addDependency(d); - - d = new Dependency(); - d.setName("voodoo"); - d.setType("custom"); - algorithm.addDependency(d); - - ***REMOVED*** create the algorithm (download it, etc etc) - Algorithm ewe = new Algorithm(); - ewe.setName("ewe"); - - d = new Dependency(); - d.setName("voodoo"); - d.setType("custom"); - ewe.addDependency(d); +***REMOVED*** +***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) +***REMOVED*** Algorithm algorithm = new Algorithm(); +***REMOVED*** algorithm.setName("ichtyop"); +***REMOVED*** algorithm.setClazz("org.gcube..."); +***REMOVED*** algorithm.setDescription("some description"); +***REMOVED*** +***REMOVED*** Dependency d = new Dependency(); +***REMOVED*** d.setName("libpng"); +***REMOVED*** d.setType("os"); +***REMOVED*** algorithm.addDependency(d); +***REMOVED*** +***REMOVED*** d = new Dependency(); +***REMOVED*** d.setName("some-r-package"); +***REMOVED*** d.setType("cran"); +***REMOVED*** algorithm.addDependency(d); +***REMOVED*** +***REMOVED*** d = new Dependency(); +***REMOVED*** d.setName("some-other-r-package"); +***REMOVED*** d.setType("cran"); +***REMOVED*** algorithm.addDependency(d); +***REMOVED*** +***REMOVED*** d = new Dependency(); +***REMOVED*** d.setName("voodoo"); +***REMOVED*** d.setType("custom"); +***REMOVED*** algorithm.addDependency(d); +***REMOVED*** +***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) +***REMOVED*** Algorithm ewe = new Algorithm(); +***REMOVED*** ewe.setName("ewe"); +***REMOVED*** +***REMOVED*** d = new Dependency(); +***REMOVED*** d.setName("voodoo"); +***REMOVED*** d.setType("custom"); +***REMOVED*** ewe.addDependency(d); AlgorithmSet algorithms = new AlgorithmSet(); algorithms.setName("dummy-set"); - algorithms.addAlgorithm(algorithm); - algorithms.addAlgorithm(ewe); +***REMOVED*** algorithms.addAlgorithm(algorithm); +***REMOVED*** algorithms.addAlgorithm(ewe); Algorithm ensemble = new Algorithm(); ensemble.setName("ensemble"); - d = new Dependency(); - d.setName("libpng"); + Dependency d = new Dependency(); + d.setName("libpng3"); d.setType("os"); ensemble.addDependency(d); algorithms.addAlgorithm(ensemble); - ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext"); + new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"); ***REMOVED***