git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148431 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
745a01d0d1
commit
9d3cf1deb6
|
|
@ -204,82 +204,82 @@ public class AnsibleBridge {
|
|||
}
|
||||
}
|
||||
|
||||
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
|
||||
|
||||
return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test);
|
||||
}
|
||||
// public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
//
|
||||
//
|
||||
// return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test);
|
||||
// }
|
||||
|
||||
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
|
||||
|
||||
|
||||
List<Role> algoRoles = new Vector<>();
|
||||
|
||||
// add algorithms and dependencies to the worker
|
||||
for (Algorithm a : as.getAlgorithms()) {
|
||||
for (Role r : this.generateRoles(a)) {
|
||||
algoRoles.add(r);
|
||||
worker.addRole(r);
|
||||
}
|
||||
//to comment the for in case of just install algo
|
||||
if(test){
|
||||
for (Dependency d : a.getDependencies()) {
|
||||
for (Role r : this.generateRoles(d)) {
|
||||
worker.addRole(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// add static roles
|
||||
for(Role r:this.getStaticRoleManager().getStaticRoles()) {
|
||||
worker.addRole(r);
|
||||
}
|
||||
|
||||
// generate the inventory
|
||||
Inventory inventory = new Inventory();
|
||||
for (Host h : cluster.getHosts()) {
|
||||
AnsibleHost ah = new AnsibleHost(h.getName());
|
||||
inventory.addHost(ah, "universe");
|
||||
inventory.addHost(ah, "d4science");
|
||||
}
|
||||
worker.setInventory(inventory);
|
||||
|
||||
// generate the playbook
|
||||
Playbook playbook = new Playbook();
|
||||
if(test){
|
||||
playbook.setRemote_user("root");}
|
||||
playbook.setRemote_user("gcube");
|
||||
playbook.applyTo("universe");
|
||||
for(Role r:algoRoles) {
|
||||
// add only 'add' roles
|
||||
if(!r.getName().endsWith("remove")) {
|
||||
playbook.addRole(r.getName());
|
||||
}
|
||||
}
|
||||
|
||||
worker.setPlaybook(playbook);
|
||||
|
||||
// execute and save log locally
|
||||
//PrintStream console = System.out;
|
||||
File path = new File(worker.getWorkdir() + File.separator + "logs");
|
||||
path.mkdirs();
|
||||
File n = new File(path + File.separator + worker.getWorkerId());
|
||||
FileOutputStream fos = new FileOutputStream(n);
|
||||
PrintStream ps = new PrintStream(fos);
|
||||
|
||||
//System.setErr(console);
|
||||
|
||||
worker.apply(as,ps,test);
|
||||
//System.setOut(console);
|
||||
//worker.apply();
|
||||
System.out.println("Log stored to to " + n.getAbsolutePath());
|
||||
|
||||
// destroy the worker
|
||||
worker.destroy();
|
||||
return worker;
|
||||
}
|
||||
// public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
// AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
|
||||
//
|
||||
//
|
||||
// List<Role> algoRoles = new Vector<>();
|
||||
//
|
||||
// // add algorithms and dependencies to the worker
|
||||
// for (Algorithm a : as.getAlgorithms()) {
|
||||
// for (Role r : this.generateRoles(a)) {
|
||||
// algoRoles.add(r);
|
||||
// worker.addRole(r);
|
||||
// }
|
||||
// //to comment the for in case of just install algo
|
||||
// if(test){
|
||||
// for (Dependency d : a.getDependencies()) {
|
||||
// for (Role r : this.generateRoles(d)) {
|
||||
// worker.addRole(r);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// // add static roles
|
||||
// for(Role r:this.getStaticRoleManager().getStaticRoles()) {
|
||||
// worker.addRole(r);
|
||||
// }
|
||||
//
|
||||
// // generate the inventory
|
||||
// Inventory inventory = new Inventory();
|
||||
// for (Host h : cluster.getHosts()) {
|
||||
// AnsibleHost ah = new AnsibleHost(h.getName());
|
||||
// inventory.addHost(ah, "universe");
|
||||
// inventory.addHost(ah, "d4science");
|
||||
// }
|
||||
// worker.setInventory(inventory);
|
||||
//
|
||||
// // generate the playbook
|
||||
// Playbook playbook = new Playbook();
|
||||
// if(test){
|
||||
// playbook.setRemote_user("root");}
|
||||
// playbook.setRemote_user("gcube");
|
||||
// playbook.applyTo("universe");
|
||||
// for(Role r:algoRoles) {
|
||||
// // add only 'add' roles
|
||||
// if(!r.getName().endsWith("remove")) {
|
||||
// playbook.addRole(r.getName());
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// worker.setPlaybook(playbook);
|
||||
//
|
||||
// // execute and save log locally
|
||||
// //PrintStream console = System.out;
|
||||
// File path = new File(worker.getWorkdir() + File.separator + "logs");
|
||||
// path.mkdirs();
|
||||
// File n = new File(path + File.separator + worker.getWorkerId());
|
||||
// FileOutputStream fos = new FileOutputStream(n);
|
||||
// PrintStream ps = new PrintStream(fos);
|
||||
//
|
||||
// //System.setErr(console);
|
||||
//
|
||||
// worker.apply(as,ps,test);
|
||||
// //System.setOut(console);
|
||||
// //worker.apply();
|
||||
// System.out.println("Log stored to to " + n.getAbsolutePath());
|
||||
//
|
||||
// // destroy the worker
|
||||
// worker.destroy();
|
||||
// return worker;
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -172,70 +172,70 @@ public class DataminerPoolManager {
|
|||
return bufferScript;
|
||||
}
|
||||
|
||||
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
||||
// create a fake algorithm set
|
||||
final AlgorithmSet algoSet = new AlgorithmSet();
|
||||
algoSet.setName("fake");
|
||||
algoSet.addAlgorithm(algorithm);
|
||||
final String uuid = UUID.randomUUID().toString();
|
||||
// public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
||||
// // create a fake algorithm set
|
||||
// final AlgorithmSet algoSet = new AlgorithmSet();
|
||||
// algoSet.setName("fake");
|
||||
// algoSet.addAlgorithm(algorithm);
|
||||
// final String uuid = UUID.randomUUID().toString();
|
||||
//
|
||||
// new Thread(new Runnable() {
|
||||
// @Override
|
||||
// public void run() {
|
||||
// // TODO Auto-generated method stub
|
||||
// try {
|
||||
// try {
|
||||
// addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
|
||||
// } catch (SVNException e) {
|
||||
// // TODO Auto-generated catch block
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// } catch (IOException e) {
|
||||
// // TODO Auto-generated catch block
|
||||
// e.printStackTrace();
|
||||
// } catch (InterruptedException e) {
|
||||
// // TODO Auto-generated catch block
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// }
|
||||
// }).start();
|
||||
// // this line will execute immediately, not waiting for task to
|
||||
// // complete
|
||||
// System.out.println(uuid);
|
||||
// return uuid;
|
||||
// }
|
||||
|
||||
new Thread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
// TODO Auto-generated method stub
|
||||
try {
|
||||
try {
|
||||
addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
|
||||
} catch (SVNException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (InterruptedException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}).start();
|
||||
// this line will execute immediately, not waiting for task to
|
||||
// complete
|
||||
System.out.println(uuid);
|
||||
return uuid;
|
||||
}
|
||||
|
||||
public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
||||
// create a fake algorithm set
|
||||
final AlgorithmSet algoSet = new AlgorithmSet();
|
||||
algoSet.setName("fake");
|
||||
algoSet.addAlgorithm(algorithm);
|
||||
final String uuid = UUID.randomUUID().toString();
|
||||
|
||||
new Thread(new Runnable() {
|
||||
@Override
|
||||
public void run() {
|
||||
// TODO Auto-generated method stub
|
||||
try {
|
||||
if(test){
|
||||
addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);}
|
||||
} catch (IOException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (InterruptedException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
} catch (SVNException e) {
|
||||
// TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
}).start();
|
||||
// this line will execute immediately, not waiting for your task to
|
||||
// complete
|
||||
System.out.println(uuid);
|
||||
return uuid;
|
||||
}
|
||||
// public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
||||
// // create a fake algorithm set
|
||||
// final AlgorithmSet algoSet = new AlgorithmSet();
|
||||
// algoSet.setName("fake");
|
||||
// algoSet.addAlgorithm(algorithm);
|
||||
// final String uuid = UUID.randomUUID().toString();
|
||||
//
|
||||
// new Thread(new Runnable() {
|
||||
// @Override
|
||||
// public void run() {
|
||||
// // TODO Auto-generated method stub
|
||||
// try {
|
||||
// if(test){
|
||||
// addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);}
|
||||
// } catch (IOException e) {
|
||||
// // TODO Auto-generated catch block
|
||||
// e.printStackTrace();
|
||||
// } catch (InterruptedException e) {
|
||||
// // TODO Auto-generated catch block
|
||||
// e.printStackTrace();
|
||||
// } catch (SVNException e) {
|
||||
// // TODO Auto-generated catch block
|
||||
// e.printStackTrace();
|
||||
// }
|
||||
// }
|
||||
// }).start();
|
||||
// // this line will execute immediately, not waiting for your task to
|
||||
// // complete
|
||||
// System.out.println(uuid);
|
||||
// return uuid;
|
||||
// }
|
||||
|
||||
|
||||
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
|
||||
|
|
@ -249,52 +249,52 @@ public class DataminerPoolManager {
|
|||
return new File(n.getPath()).toURI().toURL();
|
||||
}
|
||||
|
||||
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
// public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
||||
//
|
||||
// // create the cluster (dataminers in the vre)
|
||||
// Cluster cluster = new Cluster();
|
||||
// for (Host h : new HAProxy().listDataMinersByCluster()) {
|
||||
// //for (Host h : new ISClient().listDataminersInVRE()) {
|
||||
// cluster.addHost(h);
|
||||
// }
|
||||
//
|
||||
// // apply the changes
|
||||
// AnsibleBridge a = new AnsibleBridge();
|
||||
// return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
|
||||
//
|
||||
// }
|
||||
|
||||
// create the cluster (dataminers in the vre)
|
||||
Cluster cluster = new Cluster();
|
||||
for (Host h : new HAProxy().listDataMinersByCluster()) {
|
||||
//for (Host h : new ISClient().listDataminersInVRE()) {
|
||||
cluster.addHost(h);
|
||||
}
|
||||
|
||||
// apply the changes
|
||||
AnsibleBridge a = new AnsibleBridge();
|
||||
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
|
||||
|
||||
}
|
||||
|
||||
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
||||
throws IOException, InterruptedException, SVNException {
|
||||
|
||||
// create the cluster (dataminers in the vre)
|
||||
Cluster cluster = new Cluster();
|
||||
for (Host h : new HAProxy().listDataMinersByCluster()) {
|
||||
if (h.getName().equals(hostname)) {
|
||||
cluster.addHost(h);
|
||||
}
|
||||
}
|
||||
// if(ISClient.getHProxy().equals(hostname)){
|
||||
// cluster.addHost(new ISClient().getDataminer(hostname));
|
||||
// }
|
||||
// apply the changes
|
||||
AnsibleBridge a = new AnsibleBridge();
|
||||
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
||||
|
||||
}
|
||||
// public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
||||
// throws IOException, InterruptedException, SVNException {
|
||||
//
|
||||
// // create the cluster (dataminers in the vre)
|
||||
// Cluster cluster = new Cluster();
|
||||
// for (Host h : new HAProxy().listDataMinersByCluster()) {
|
||||
// if (h.getName().equals(hostname)) {
|
||||
// cluster.addHost(h);
|
||||
// }
|
||||
// }
|
||||
// // if(ISClient.getHProxy().equals(hostname)){
|
||||
// // cluster.addHost(new ISClient().getDataminer(hostname));
|
||||
// // }
|
||||
// // apply the changes
|
||||
// AnsibleBridge a = new AnsibleBridge();
|
||||
// return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
||||
throws IOException, InterruptedException, SVNException {
|
||||
Cluster cluster = new Cluster();
|
||||
Host h = new Host();
|
||||
h.setName(hostname);
|
||||
cluster.addHost(h);
|
||||
|
||||
AnsibleBridge a = new AnsibleBridge();
|
||||
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
||||
|
||||
}
|
||||
//
|
||||
// public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
||||
// throws IOException, InterruptedException, SVNException {
|
||||
// Cluster cluster = new Cluster();
|
||||
// Host h = new Host();
|
||||
// h.setName(hostname);
|
||||
// cluster.addHost(h);
|
||||
//
|
||||
// AnsibleBridge a = new AnsibleBridge();
|
||||
// return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
||||
//
|
||||
// }
|
||||
|
||||
|
||||
// 2017 March 29
|
||||
|
|
|
|||
|
|
@ -23,29 +23,22 @@ public class AlgorithmBuilder {
|
|||
if(category != null){
|
||||
algo.setCategory(category);
|
||||
}
|
||||
|
||||
//FIXME: do the same done for category
|
||||
if (algo.getAlgorithmType() == null) {
|
||||
algo.setAlgorithmType(algorithmType);
|
||||
} else
|
||||
algo.setAlgorithmType(algo.getCategory());
|
||||
|
||||
if (algo.getSkipJava() == null) {
|
||||
algo.setSkipJava(skipJava);
|
||||
} else
|
||||
algo.setSkipJava(algo.getSkipJava());
|
||||
|
||||
if (algo.getName() == null) {
|
||||
algo.setName(name);
|
||||
} else
|
||||
algo.setName(algo.getName());
|
||||
|
||||
if (algo.getDescription() == null) {
|
||||
algo.setDescription(description);
|
||||
;
|
||||
} else
|
||||
algo.setDescription(algo.getDescription());
|
||||
|
||||
if(algorithmType != null){
|
||||
algo.setAlgorithmType(algorithmType);
|
||||
}
|
||||
if(skipJava != null){
|
||||
algo.setSkipJava(skipJava);
|
||||
}
|
||||
if(skipJava != null){
|
||||
algo.setSkipJava(skipJava);
|
||||
}
|
||||
if(name != null){
|
||||
algo.setName(name);
|
||||
}
|
||||
if(description != null){
|
||||
algo.setDescription(description);
|
||||
}
|
||||
|
||||
return algo;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -220,7 +220,7 @@ public class DataminerPoolManagerTest {
|
|||
ensemble.addDependency(d);
|
||||
algorithms.addAlgorithm(ensemble);
|
||||
|
||||
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
|
||||
//new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -220,7 +220,7 @@ public class DataminerPoolManagerTest {
|
|||
ensemble.addDependency(d);
|
||||
algorithms.addAlgorithm(ensemble);
|
||||
|
||||
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
|
||||
//new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
|
||||
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue