gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@144015 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-02-21 13:42:26 +00:00
parent d699f64c51
commit 87218738ec
8 changed files with 97 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
View File

@ -3,8 +3,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible;
***REMOVED***
import java.io.File;
***REMOVED***
import java.io.InputStream;
***REMOVED***
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Scanner;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
@ -65,7 +67,7 @@ public class AnsibleWorker {
public void removeWorkStructure() {
***REMOVED*** remove the working dir
***REMOVED***this.getWorkdir().delete();
this.getWorkdir().delete();
***REMOVED***
public File getPlaybookFile() {
@ -96,21 +98,32 @@ public class AnsibleWorker {
public void apply() throws IOException {
***REMOVED*** TODO execute the playbook and return output
System.out.println(this.getWorkdir());
try {
System.out.println("ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
BufferedReader in = new BufferedReader(
new InputStreamReader(p.getInputStream()));
String line = null;
while ((line = in.readLine()) != null) {
System.out.println(line);
***REMOVED***
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath());
inheritIO(p.getInputStream(), System.out);
inheritIO(p.getErrorStream(), System.err);
***REMOVED*** catch (IOException e) {
e.printStackTrace();
***REMOVED***
***REMOVED***System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
***REMOVED***
private static void inheritIO(final InputStream src, final PrintStream dest) {
new Thread(new Runnable() {
public void run() {
Scanner sc = new Scanner(src);
while (sc.hasNextLine()) {
dest.println(sc.nextLine());
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED***
/**
* Destroy the worker:
* - remove the working dir

18
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java
View File

@ -9,6 +9,8 @@ public class Playbook {
private List<String> roles;
private String remote_user;
public Playbook() {
this.roles = new Vector<>();
***REMOVED***
@ -28,5 +30,21 @@ public class Playbook {
public List<String> getRoles() {
return new Vector<>(roles);
***REMOVED***
public String getRemote_user() {
return remote_user;
***REMOVED***
public void setRemote_user(String remote_user) {
this.remote_user = remote_user;
***REMOVED***
public void setHostGroupName(String hostGroupName) {
this.hostGroupName = hostGroupName;
***REMOVED***
public void setRoles(List<String> roles) {
this.roles = roles;
***REMOVED***
***REMOVED***

1
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -184,6 +184,7 @@ public class AnsibleBridge {
***REMOVED*** generate the playbook
Playbook playbook = new Playbook();
playbook.setRemote_user("root");
playbook.applyTo("universe");
for(Role r:algoRoles) {
***REMOVED*** add only 'add' roles

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java
View File

@ -31,7 +31,7 @@ public class AnsibleSerializeHelper {
public static void serialize(Playbook playbook, File playbookFile) throws IOException {
String out = "- hosts: " + playbook.getHostGroupName() + "\n";
out += " remote_user: dpm\n";
out += " remote_user: "+playbook.getRemote_user()+"\n";
out+=" roles:\n";
for(String r:playbook.getRoles()) {
out+=" - " + r+"\n";

17
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -37,9 +37,12 @@ public class ISClient {
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
.addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
***REMOVED***old version
***REMOVED***query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
***REMOVED***.addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> resources = client.submit(query);
@ -54,9 +57,9 @@ public class ISClient {
***REMOVED***
***REMOVED***
***REMOVED*** public static void main(String[] args) {
***REMOVED*** ISClient a = new ISClient();
***REMOVED*** ScopeProvider.instance.set("/gcube/devNext/NextNext");
***REMOVED*** System.out.println(a.listDataminersInVRE());
***REMOVED******REMOVED***
public static void main(String[] args) {
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(a.listDataminersInVRE());
***REMOVED***
***REMOVED***

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -59,7 +59,7 @@ public class RestPoolManager implements PoolManager {
public static void main(String[] args) throws IOException, InterruptedException {
RestPoolManager a = new RestPoolManager();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
a.addAlgorithmToVRE("http:***REMOVED***data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0", "/gcube/devNext/NextNext");
a.addAlgorithmToVRE("http:***REMOVED***data.d4science.org/eDF4dWtTc000dEpSLzcybU4zRmJoTmFwUnhZeGZBN1dHbWJQNStIS0N6Yz0", "/gcube/devNext/NextNext");
***REMOVED***System.out.println(a.getLogById("dadcb059-69e5-48c3-aa58-3b290ae0419d"));
***REMOVED***

5
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -193,7 +193,7 @@ public class DataminerPoolManager implements PoolManager {
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
Runnable r = new Runnable() {
new Thread(new Runnable() {
@Override
public void run() {
***REMOVED*** TODO Auto-generated method stub
@ -204,8 +204,7 @@ public class DataminerPoolManager implements PoolManager {
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***;
new Thread(r).start();
***REMOVED***).start();
***REMOVED***this line will execute immediately, not waiting for your task to complete
System.out.println(uuid);
return uuid;

81
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java
View File

@ -6,6 +6,7 @@ import java.net.ProxySelector;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class DataminerPoolManagerTest {
@ -160,58 +161,58 @@ public class DataminerPoolManagerTest {
ScopeProvider.instance.set("/gcube/devNext/NextNext");
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
***REMOVED*** create the algorithm (download it, etc etc)
Algorithm algorithm = new Algorithm();
algorithm.setName("ichtyop");
algorithm.setClazz("org.gcube...");
algorithm.setDescription("some description");
Dependency d = new Dependency();
d.setName("libpng");
d.setType("os");
algorithm.addDependency(d);
d = new Dependency();
d.setName("some-r-package");
d.setType("cran");
algorithm.addDependency(d);
d = new Dependency();
d.setName("some-other-r-package");
d.setType("cran");
algorithm.addDependency(d);
d = new Dependency();
d.setName("voodoo");
d.setType("custom");
algorithm.addDependency(d);
***REMOVED*** create the algorithm (download it, etc etc)
Algorithm ewe = new Algorithm();
ewe.setName("ewe");
d = new Dependency();
d.setName("voodoo");
d.setType("custom");
ewe.addDependency(d);
***REMOVED***
***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc)
***REMOVED*** Algorithm algorithm = new Algorithm();
***REMOVED*** algorithm.setName("ichtyop");
***REMOVED*** algorithm.setClazz("org.gcube...");
***REMOVED*** algorithm.setDescription("some description");
***REMOVED***
***REMOVED*** Dependency d = new Dependency();
***REMOVED*** d.setName("libpng");
***REMOVED*** d.setType("os");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("some-r-package");
***REMOVED*** d.setType("cran");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("some-other-r-package");
***REMOVED*** d.setType("cran");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("voodoo");
***REMOVED*** d.setType("custom");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc)
***REMOVED*** Algorithm ewe = new Algorithm();
***REMOVED*** ewe.setName("ewe");
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("voodoo");
***REMOVED*** d.setType("custom");
***REMOVED*** ewe.addDependency(d);
AlgorithmSet algorithms = new AlgorithmSet();
algorithms.setName("dummy-set");
algorithms.addAlgorithm(algorithm);
algorithms.addAlgorithm(ewe);
***REMOVED*** algorithms.addAlgorithm(algorithm);
***REMOVED*** algorithms.addAlgorithm(ewe);
Algorithm ensemble = new Algorithm();
ensemble.setName("ensemble");
d = new Dependency();
d.setName("libpng");
Dependency d = new Dependency();
d.setName("libpng3");
d.setType("os");
ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble);
***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext");
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test");
***REMOVED***