gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@144015 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-02-21 13:42:26 +00:00
parent 6c282d1754
commit b7fec5b2eb
8 changed files with 97 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
View File

@ -3,8 +3,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.util.Scanner;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
@ -65,7 +67,7 @@ public class AnsibleWorker {
public void removeWorkStructure() {
// remove the working dir
//this.getWorkdir().delete();
this.getWorkdir().delete();
}
public File getPlaybookFile() {
@ -96,21 +98,32 @@ public class AnsibleWorker {
public void apply() throws IOException {
// TODO execute the playbook and return output
System.out.println(this.getWorkdir());
try {
System.out.println("ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
BufferedReader in = new BufferedReader(
new InputStreamReader(p.getInputStream()));
String line = null;
while ((line = in.readLine()) != null) {
System.out.println(line);
}
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath());
inheritIO(p.getInputStream(), System.out);
inheritIO(p.getErrorStream(), System.err);
} catch (IOException e) {
e.printStackTrace();
}
//System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
}
private static void inheritIO(final InputStream src, final PrintStream dest) {
new Thread(new Runnable() {
public void run() {
Scanner sc = new Scanner(src);
while (sc.hasNextLine()) {
dest.println(sc.nextLine());
}
}
}).start();
}
/**
* Destroy the worker:
* - remove the working dir

18
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java
View File

@ -9,6 +9,8 @@ public class Playbook {
private List<String> roles;
private String remote_user;
public Playbook() {
this.roles = new Vector<>();
}
@ -28,5 +30,21 @@ public class Playbook {
public List<String> getRoles() {
return new Vector<>(roles);
}
public String getRemote_user() {
return remote_user;
}
public void setRemote_user(String remote_user) {
this.remote_user = remote_user;
}
public void setHostGroupName(String hostGroupName) {
this.hostGroupName = hostGroupName;
}
public void setRoles(List<String> roles) {
this.roles = roles;
}
}

1
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -184,6 +184,7 @@ public class AnsibleBridge {
// generate the playbook
Playbook playbook = new Playbook();
playbook.setRemote_user("root");
playbook.applyTo("universe");
for(Role r:algoRoles) {
// add only 'add' roles

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java
View File

@ -31,7 +31,7 @@ public class AnsibleSerializeHelper {
public static void serialize(Playbook playbook, File playbookFile) throws IOException {
String out = "- hosts: " + playbook.getHostGroupName() + "\n";
out += " remote_user: dpm\n";
out += " remote_user: "+playbook.getRemote_user()+"\n";
out+=" roles:\n";
for(String r:playbook.getRoles()) {
out+=" - " + r+"\n";

17
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -37,9 +37,12 @@ public class ISClient {
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
.addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
//old version
//query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
//.addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> resources = client.submit(query);
@ -54,9 +57,9 @@ public class ISClient {
}
}
// public static void main(String[] args) {
// ISClient a = new ISClient();
// ScopeProvider.instance.set("/gcube/devNext/NextNext");
// System.out.println(a.listDataminersInVRE());
//}
public static void main(String[] args) {
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(a.listDataminersInVRE());
}
}

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -59,7 +59,7 @@ public class RestPoolManager implements PoolManager {
public static void main(String[] args) throws IOException, InterruptedException {
RestPoolManager a = new RestPoolManager();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
a.addAlgorithmToVRE("http://data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0", "/gcube/devNext/NextNext");
a.addAlgorithmToVRE("http://data.d4science.org/eDF4dWtTc000dEpSLzcybU4zRmJoTmFwUnhZeGZBN1dHbWJQNStIS0N6Yz0", "/gcube/devNext/NextNext");
//System.out.println(a.getLogById("dadcb059-69e5-48c3-aa58-3b290ae0419d"));
}

5
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -193,7 +193,7 @@ public class DataminerPoolManager implements PoolManager {
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
Runnable r = new Runnable() {
new Thread(new Runnable() {
@Override
public void run() {
// TODO Auto-generated method stub
@ -204,8 +204,7 @@ public class DataminerPoolManager implements PoolManager {
e.printStackTrace();
}
}
};
new Thread(r).start();
}).start();
//this line will execute immediately, not waiting for your task to complete
System.out.println(uuid);
return uuid;

81
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java
View File

@ -6,6 +6,7 @@ import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class DataminerPoolManagerTest {
@ -160,58 +161,58 @@ public class DataminerPoolManagerTest {
ScopeProvider.instance.set("/gcube/devNext/NextNext");
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
// create the algorithm (download it, etc etc)
Algorithm algorithm = new Algorithm();
algorithm.setName("ichtyop");
algorithm.setClazz("org.gcube...");
algorithm.setDescription("some description");
Dependency d = new Dependency();
d.setName("libpng");
d.setType("os");
algorithm.addDependency(d);
d = new Dependency();
d.setName("some-r-package");
d.setType("cran");
algorithm.addDependency(d);
d = new Dependency();
d.setName("some-other-r-package");
d.setType("cran");
algorithm.addDependency(d);
d = new Dependency();
d.setName("voodoo");
d.setType("custom");
algorithm.addDependency(d);
// create the algorithm (download it, etc etc)
Algorithm ewe = new Algorithm();
ewe.setName("ewe");
d = new Dependency();
d.setName("voodoo");
d.setType("custom");
ewe.addDependency(d);
//
// // create the algorithm (download it, etc etc)
// Algorithm algorithm = new Algorithm();
// algorithm.setName("ichtyop");
// algorithm.setClazz("org.gcube...");
// algorithm.setDescription("some description");
//
// Dependency d = new Dependency();
// d.setName("libpng");
// d.setType("os");
// algorithm.addDependency(d);
//
// d = new Dependency();
// d.setName("some-r-package");
// d.setType("cran");
// algorithm.addDependency(d);
//
// d = new Dependency();
// d.setName("some-other-r-package");
// d.setType("cran");
// algorithm.addDependency(d);
//
// d = new Dependency();
// d.setName("voodoo");
// d.setType("custom");
// algorithm.addDependency(d);
//
// // create the algorithm (download it, etc etc)
// Algorithm ewe = new Algorithm();
// ewe.setName("ewe");
//
// d = new Dependency();
// d.setName("voodoo");
// d.setType("custom");
// ewe.addDependency(d);
AlgorithmSet algorithms = new AlgorithmSet();
algorithms.setName("dummy-set");
algorithms.addAlgorithm(algorithm);
algorithms.addAlgorithm(ewe);
// algorithms.addAlgorithm(algorithm);
// algorithms.addAlgorithm(ewe);
Algorithm ensemble = new Algorithm();
ensemble.setName("ensemble");
d = new Dependency();
d.setName("libpng");
Dependency d = new Dependency();
d.setName("libpng3");
d.setType("os");
ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble);
//new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext");
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test");
}