gCubeSystem
/
dataminer-pool-manager
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148431 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-05-10 09:31:28 +00:00
parent e136299fb9
commit a325272b18
5 changed files with 196 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -204,82 +204,82 @@ public class AnsibleBridge {
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { ***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
***REMOVED***
***REMOVED***
***REMOVED*** return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test);
***REMOVED*** ***REMOVED***
***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test); ***REMOVED*** AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
***REMOVED*** ***REMOVED***
***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { ***REMOVED*** List<Role> algoRoles = new Vector<>();
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); ***REMOVED***
***REMOVED*** ***REMOVED*** add algorithms and dependencies to the worker
***REMOVED*** for (Algorithm a : as.getAlgorithms()) {
List<Role> algoRoles = new Vector<>(); ***REMOVED*** for (Role r : this.generateRoles(a)) {
***REMOVED*** algoRoles.add(r);
***REMOVED*** add algorithms and dependencies to the worker ***REMOVED*** worker.addRole(r);
for (Algorithm a : as.getAlgorithms()) { ***REMOVED*** ***REMOVED***
for (Role r : this.generateRoles(a)) { ***REMOVED*** ***REMOVED***to comment the for in case of just install algo
algoRoles.add(r); ***REMOVED*** if(test){
worker.addRole(r); ***REMOVED*** for (Dependency d : a.getDependencies()) {
***REMOVED*** ***REMOVED*** for (Role r : this.generateRoles(d)) {
***REMOVED***to comment the for in case of just install algo ***REMOVED*** worker.addRole(r);
if(test){ ***REMOVED*** ***REMOVED***
for (Dependency d : a.getDependencies()) { ***REMOVED*** ***REMOVED***
for (Role r : this.generateRoles(d)) { ***REMOVED*** ***REMOVED***
worker.addRole(r); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** ***REMOVED*** add static roles
***REMOVED*** ***REMOVED*** for(Role r:this.getStaticRoleManager().getStaticRoles()) {
***REMOVED*** ***REMOVED*** worker.addRole(r);
***REMOVED*** ***REMOVED***
***REMOVED*** add static roles ***REMOVED***
for(Role r:this.getStaticRoleManager().getStaticRoles()) { ***REMOVED*** ***REMOVED*** generate the inventory
worker.addRole(r); ***REMOVED*** Inventory inventory = new Inventory();
***REMOVED*** ***REMOVED*** for (Host h : cluster.getHosts()) {
***REMOVED*** AnsibleHost ah = new AnsibleHost(h.getName());
***REMOVED*** generate the inventory ***REMOVED*** inventory.addHost(ah, "universe");
Inventory inventory = new Inventory(); ***REMOVED*** inventory.addHost(ah, "d4science");
for (Host h : cluster.getHosts()) { ***REMOVED*** ***REMOVED***
AnsibleHost ah = new AnsibleHost(h.getName()); ***REMOVED*** worker.setInventory(inventory);
inventory.addHost(ah, "universe"); ***REMOVED***
inventory.addHost(ah, "d4science"); ***REMOVED*** ***REMOVED*** generate the playbook
***REMOVED*** ***REMOVED*** Playbook playbook = new Playbook();
worker.setInventory(inventory); ***REMOVED*** if(test){
***REMOVED*** playbook.setRemote_user("root");***REMOVED***
***REMOVED*** generate the playbook ***REMOVED*** playbook.setRemote_user("gcube");
Playbook playbook = new Playbook(); ***REMOVED*** playbook.applyTo("universe");
if(test){ ***REMOVED*** for(Role r:algoRoles) {
playbook.setRemote_user("root");***REMOVED*** ***REMOVED*** ***REMOVED*** add only 'add' roles
playbook.setRemote_user("gcube"); ***REMOVED*** if(!r.getName().endsWith("remove")) {
playbook.applyTo("universe"); ***REMOVED*** playbook.addRole(r.getName());
for(Role r:algoRoles) { ***REMOVED*** ***REMOVED***
***REMOVED*** add only 'add' roles ***REMOVED*** ***REMOVED***
if(!r.getName().endsWith("remove")) { ***REMOVED***
playbook.addRole(r.getName()); ***REMOVED*** worker.setPlaybook(playbook);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** ***REMOVED*** execute and save log locally
***REMOVED*** ***REMOVED***PrintStream console = System.out;
worker.setPlaybook(playbook); ***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
***REMOVED*** path.mkdirs();
***REMOVED*** execute and save log locally ***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
***REMOVED***PrintStream console = System.out; ***REMOVED*** FileOutputStream fos = new FileOutputStream(n);
File path = new File(worker.getWorkdir() + File.separator + "logs"); ***REMOVED*** PrintStream ps = new PrintStream(fos);
path.mkdirs(); ***REMOVED***
File n = new File(path + File.separator + worker.getWorkerId()); ***REMOVED*** ***REMOVED***System.setErr(console);
FileOutputStream fos = new FileOutputStream(n); ***REMOVED***
PrintStream ps = new PrintStream(fos); ***REMOVED*** worker.apply(as,ps,test);
***REMOVED*** ***REMOVED***System.setOut(console);
***REMOVED***System.setErr(console); ***REMOVED*** ***REMOVED***worker.apply();
***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
worker.apply(as,ps,test); ***REMOVED***
***REMOVED***System.setOut(console); ***REMOVED*** ***REMOVED*** destroy the worker
***REMOVED***worker.apply(); ***REMOVED*** worker.destroy();
System.out.println("Log stored to to " + n.getAbsolutePath()); ***REMOVED*** return worker;
***REMOVED*** ***REMOVED***
***REMOVED*** destroy the worker
worker.destroy();
return worker;
***REMOVED***

204
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -172,70 +172,70 @@ public class DataminerPoolManager {
return bufferScript; return bufferScript;
***REMOVED*** ***REMOVED***
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException { ***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
***REMOVED*** create a fake algorithm set ***REMOVED*** ***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet(); ***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake"); ***REMOVED*** algoSet.setName("fake");
algoSet.addAlgorithm(algorithm); ***REMOVED*** algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString(); ***REMOVED*** final String uuid = UUID.randomUUID().toString();
new Thread(new Runnable() {
@Override
public void run() {
***REMOVED*** TODO Auto-generated method stub
try {
try {
addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
***REMOVED*** catch (SVNException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED*** catch (InterruptedException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED*** this line will execute immediately, not waiting for task to
***REMOVED*** complete
System.out.println(uuid);
return uuid;
***REMOVED*** ***REMOVED***
***REMOVED*** new Thread(new Runnable() {
***REMOVED*** @Override
***REMOVED*** public void run() {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** try {
***REMOVED*** try {
***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
***REMOVED*** ***REMOVED*** catch (SVNException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** catch (IOException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED*** catch (InterruptedException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***).start();
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for task to
***REMOVED*** ***REMOVED*** complete
***REMOVED*** System.out.println(uuid);
***REMOVED*** return uuid;
***REMOVED******REMOVED***
public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException { ***REMOVED*** public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
***REMOVED*** create a fake algorithm set ***REMOVED*** ***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet(); ***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake"); ***REMOVED*** algoSet.setName("fake");
algoSet.addAlgorithm(algorithm); ***REMOVED*** algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString(); ***REMOVED*** final String uuid = UUID.randomUUID().toString();
new Thread(new Runnable() {
@Override
public void run() {
***REMOVED*** TODO Auto-generated method stub
try {
if(test){
addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED*** catch (InterruptedException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED*** catch (SVNException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED*** this line will execute immediately, not waiting for your task to
***REMOVED*** complete
System.out.println(uuid);
return uuid;
***REMOVED*** ***REMOVED***
***REMOVED*** new Thread(new Runnable() {
***REMOVED*** @Override
***REMOVED*** public void run() {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** try {
***REMOVED*** if(test){
***REMOVED*** addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
***REMOVED*** ***REMOVED*** catch (IOException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED*** catch (InterruptedException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED*** catch (SVNException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***).start();
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for your task to
***REMOVED*** ***REMOVED*** complete
***REMOVED*** System.out.println(uuid);
***REMOVED*** return uuid;
***REMOVED******REMOVED***
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException { public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
@ -249,52 +249,52 @@ public class DataminerPoolManager {
return new File(n.getPath()).toURI().toURL(); return new File(n.getPath()).toURI().toURL();
***REMOVED*** ***REMOVED***
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { ***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
***REMOVED***
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
***REMOVED*** Cluster cluster = new Cluster();
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) {
***REMOVED*** ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) {
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED*** apply the changes
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** create the cluster (dataminers in the vre) ***REMOVED*** public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
Cluster cluster = new Cluster(); ***REMOVED*** throws IOException, InterruptedException, SVNException {
for (Host h : new HAProxy().listDataMinersByCluster()) { ***REMOVED***
***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) { ***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
cluster.addHost(h); ***REMOVED*** Cluster cluster = new Cluster();
***REMOVED*** ***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) {
***REMOVED*** if (h.getName().equals(hostname)) {
***REMOVED*** apply the changes ***REMOVED*** cluster.addHost(h);
AnsibleBridge a = new AnsibleBridge(); ***REMOVED*** ***REMOVED***
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId(); ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if(ISClient.getHProxy().equals(hostname)){
***REMOVED*** ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname));
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** apply the changes
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) ***REMOVED*** throws IOException, InterruptedException, SVNException {
throws IOException, InterruptedException, SVNException { ***REMOVED*** Cluster cluster = new Cluster();
***REMOVED*** Host h = new Host();
***REMOVED*** create the cluster (dataminers in the vre) ***REMOVED*** h.setName(hostname);
Cluster cluster = new Cluster(); ***REMOVED*** cluster.addHost(h);
for (Host h : new HAProxy().listDataMinersByCluster()) {
if (h.getName().equals(hostname)) {
cluster.addHost(h);
***REMOVED***
***REMOVED***
***REMOVED*** if(ISClient.getHProxy().equals(hostname)){
***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname));
***REMOVED*** ***REMOVED***
***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED*** ***REMOVED***
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
throws IOException, InterruptedException, SVNException {
Cluster cluster = new Cluster();
Host h = new Host();
h.setName(hostname);
cluster.addHost(h);
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** 2017 March 29 ***REMOVED*** 2017 March 29

37
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java
View File

@ -23,28 +23,21 @@ public class AlgorithmBuilder {
if(category != null){ if(category != null){
algo.setCategory(category); algo.setCategory(category);
***REMOVED*** ***REMOVED***
if(algorithmType != null){
***REMOVED***FIXME: do the same done for category algo.setAlgorithmType(algorithmType);
if (algo.getAlgorithmType() == null) { ***REMOVED***
algo.setAlgorithmType(algorithmType); if(skipJava != null){
***REMOVED*** else algo.setSkipJava(skipJava);
algo.setAlgorithmType(algo.getCategory()); ***REMOVED***
if(skipJava != null){
if (algo.getSkipJava() == null) { algo.setSkipJava(skipJava);
algo.setSkipJava(skipJava); ***REMOVED***
***REMOVED*** else if(name != null){
algo.setSkipJava(algo.getSkipJava()); algo.setName(name);
***REMOVED***
if (algo.getName() == null) { if(description != null){
algo.setName(name); algo.setDescription(description);
***REMOVED*** else ***REMOVED***
algo.setName(algo.getName());
if (algo.getDescription() == null) {
algo.setDescription(description);
;
***REMOVED*** else
algo.setDescription(algo.getDescription());
return algo; return algo;
***REMOVED*** ***REMOVED***

2
src/test/java/DataminerPoolManagerTest.java
View File

@ -220,7 +220,7 @@ public class DataminerPoolManagerTest {
ensemble.addDependency(d); ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble); algorithms.addAlgorithm(ensemble);
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
***REMOVED*** ***REMOVED***

2
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java
View File

@ -220,7 +220,7 @@ public class DataminerPoolManagerTest {
ensemble.addDependency(d); ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble); algorithms.addAlgorithm(ensemble);
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
***REMOVED*** ***REMOVED***