gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@144015 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-02-21 13:42:26 +00:00
parent 1c77915423
commit badf193963
7 changed files with 96 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
View File

@ -3,8 +3,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible;
import java.io.BufferedReader;
import java.io.File;
***REMOVED***
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Scanner;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
@ -65,7 +67,7 @@ public class AnsibleWorker ***REMOVED***
public void removeWorkStructure() ***REMOVED***
***REMOVED*** remove the working dir
***REMOVED***this.getWorkdir().delete();
this.getWorkdir().delete();
***REMOVED***
public File getPlaybookFile() ***REMOVED***
@ -96,21 +98,32 @@ public class AnsibleWorker ***REMOVED***
public void apply() throws IOException ***REMOVED***
***REMOVED*** TODO execute the playbook and return output
System.out.println(this.getWorkdir());
try ***REMOVED***
System.out.println("ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
BufferedReader in = new BufferedReader(
new InputStreamReader(p.getInputStream()));
String line = null;
while ((line = in.readLine()) != null) ***REMOVED***
System.out.println(line);
***REMOVED***
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath());
inheritIO(p.getInputStream(), System.out);
inheritIO(p.getErrorStream(), System.err);
***REMOVED*** catch (IOException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
***REMOVED***
private static void inheritIO(final InputStream src, final PrintStream dest) ***REMOVED***
new Thread(new Runnable() ***REMOVED***
public void run() ***REMOVED***
Scanner sc = new Scanner(src);
while (sc.hasNextLine()) ***REMOVED***
dest.println(sc.nextLine());
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED***
/**
* Destroy the worker:
* - remove the working dir

18
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java
View File

@ -9,6 +9,8 @@ public class Playbook ***REMOVED***
private List<String> roles;
private String remote_user;
public Playbook() ***REMOVED***
this.roles = new Vector<>();
***REMOVED***
@ -28,5 +30,21 @@ public class Playbook ***REMOVED***
public List<String> getRoles() ***REMOVED***
return new Vector<>(roles);
***REMOVED***
public String getRemote_user() ***REMOVED***
return remote_user;
***REMOVED***
public void setRemote_user(String remote_user) ***REMOVED***
this.remote_user = remote_user;
***REMOVED***
public void setHostGroupName(String hostGroupName) ***REMOVED***
this.hostGroupName = hostGroupName;
***REMOVED***
public void setRoles(List<String> roles) ***REMOVED***
this.roles = roles;
***REMOVED***
***REMOVED***

1
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -184,6 +184,7 @@ public class AnsibleBridge ***REMOVED***
***REMOVED*** generate the playbook
Playbook playbook = new Playbook();
playbook.setRemote_user("root");
playbook.applyTo("universe");
for(Role r:algoRoles) ***REMOVED***
***REMOVED*** add only 'add' roles

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java
View File

@ -31,7 +31,7 @@ public class AnsibleSerializeHelper ***REMOVED***
public static void serialize(Playbook playbook, File playbookFile) throws IOException ***REMOVED***
String out = "- hosts: " + playbook.getHostGroupName() + "\n";
out += " remote_user: dpm\n";
out += " remote_user: "+playbook.getRemote_user()+"\n";
out+=" roles:\n";
for(String r:playbook.getRoles()) ***REMOVED***
out+=" - " + r+"\n";

17
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -37,9 +37,12 @@ public class ISClient ***REMOVED***
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
.addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
***REMOVED***old version
***REMOVED***query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
***REMOVED***.addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> resources = client.submit(query);
@ -54,9 +57,9 @@ public class ISClient ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public static void main(String[] args) ***REMOVED***
***REMOVED*** ISClient a = new ISClient();
***REMOVED*** ScopeProvider.instance.set("/gcube/devNext/NextNext");
***REMOVED*** System.out.println(a.listDataminersInVRE());
***REMOVED******REMOVED***
public static void main(String[] args) ***REMOVED***
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(a.listDataminersInVRE());
***REMOVED***
***REMOVED***

5
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -193,7 +193,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
Runnable r = new Runnable() ***REMOVED***
new Thread(new Runnable() ***REMOVED***
***REMOVED***
public void run() ***REMOVED***
***REMOVED***
@ -204,8 +204,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***;
new Thread(r).start();
***REMOVED***).start();
***REMOVED***this line will execute immediately, not waiting for your task to complete
System.out.println(uuid);
return uuid;

81
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java
View File

@ -6,6 +6,7 @@ import java.net.ProxySelector;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class DataminerPoolManagerTest ***REMOVED***
@ -160,58 +161,58 @@ public class DataminerPoolManagerTest ***REMOVED***
ScopeProvider.instance.set("/gcube/devNext/NextNext");
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
***REMOVED*** create the algorithm (download it, etc etc)
Algorithm algorithm = new Algorithm();
algorithm.setName("ichtyop");
algorithm.setClazz("org.gcube...");
algorithm.setDescription("some description");
Dependency d = new Dependency();
d.setName("libpng");
d.setType("os");
algorithm.addDependency(d);
d = new Dependency();
d.setName("some-r-package");
d.setType("cran");
algorithm.addDependency(d);
d = new Dependency();
d.setName("some-other-r-package");
d.setType("cran");
algorithm.addDependency(d);
d = new Dependency();
d.setName("voodoo");
d.setType("custom");
algorithm.addDependency(d);
***REMOVED*** create the algorithm (download it, etc etc)
Algorithm ewe = new Algorithm();
ewe.setName("ewe");
d = new Dependency();
d.setName("voodoo");
d.setType("custom");
ewe.addDependency(d);
***REMOVED***
***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc)
***REMOVED*** Algorithm algorithm = new Algorithm();
***REMOVED*** algorithm.setName("ichtyop");
***REMOVED*** algorithm.setClazz("org.gcube...");
***REMOVED*** algorithm.setDescription("some description");
***REMOVED***
***REMOVED*** Dependency d = new Dependency();
***REMOVED*** d.setName("libpng");
***REMOVED*** d.setType("os");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("some-r-package");
***REMOVED*** d.setType("cran");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("some-other-r-package");
***REMOVED*** d.setType("cran");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("voodoo");
***REMOVED*** d.setType("custom");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc)
***REMOVED*** Algorithm ewe = new Algorithm();
***REMOVED*** ewe.setName("ewe");
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("voodoo");
***REMOVED*** d.setType("custom");
***REMOVED*** ewe.addDependency(d);
AlgorithmSet algorithms = new AlgorithmSet();
algorithms.setName("dummy-set");
algorithms.addAlgorithm(algorithm);
algorithms.addAlgorithm(ewe);
***REMOVED*** algorithms.addAlgorithm(algorithm);
***REMOVED*** algorithms.addAlgorithm(ewe);
Algorithm ensemble = new Algorithm();
ensemble.setName("ensemble");
d = new Dependency();
d.setName("libpng");
Dependency d = new Dependency();
d.setName("libpng3");
d.setType("os");
ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble);
***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext");
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test");
***REMOVED***