gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148431 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-05-10 09:31:28 +00:00
parent e136299fb9
commit a325272b18
5 changed files with 196 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

150
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -204,82 +204,82 @@ public class AnsibleBridge {
***REMOVED***
***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test);
***REMOVED***
***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
***REMOVED***
***REMOVED***
***REMOVED*** return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),/*updateSVN,*/ test);
***REMOVED*** ***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
List<Role> algoRoles = new Vector<>();
***REMOVED*** add algorithms and dependencies to the worker
for (Algorithm a : as.getAlgorithms()) {
for (Role r : this.generateRoles(a)) {
algoRoles.add(r);
worker.addRole(r);
***REMOVED***
***REMOVED***to comment the for in case of just install algo
if(test){
for (Dependency d : a.getDependencies()) {
for (Role r : this.generateRoles(d)) {
worker.addRole(r);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** add static roles
for(Role r:this.getStaticRoleManager().getStaticRoles()) {
worker.addRole(r);
***REMOVED***
***REMOVED*** generate the inventory
Inventory inventory = new Inventory();
for (Host h : cluster.getHosts()) {
AnsibleHost ah = new AnsibleHost(h.getName());
inventory.addHost(ah, "universe");
inventory.addHost(ah, "d4science");
***REMOVED***
worker.setInventory(inventory);
***REMOVED*** generate the playbook
Playbook playbook = new Playbook();
if(test){
playbook.setRemote_user("root");***REMOVED***
playbook.setRemote_user("gcube");
playbook.applyTo("universe");
for(Role r:algoRoles) {
***REMOVED*** add only 'add' roles
if(!r.getName().endsWith("remove")) {
playbook.addRole(r.getName());
***REMOVED***
***REMOVED***
worker.setPlaybook(playbook);
***REMOVED*** execute and save log locally
***REMOVED***PrintStream console = System.out;
File path = new File(worker.getWorkdir() + File.separator + "logs");
path.mkdirs();
File n = new File(path + File.separator + worker.getWorkerId());
FileOutputStream fos = new FileOutputStream(n);
PrintStream ps = new PrintStream(fos);
***REMOVED***System.setErr(console);
worker.apply(as,ps,test);
***REMOVED***System.setOut(console);
***REMOVED***worker.apply();
System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED*** destroy the worker
worker.destroy();
return worker;
***REMOVED***
***REMOVED*** public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
***REMOVED*** AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
***REMOVED***
***REMOVED***
***REMOVED*** List<Role> algoRoles = new Vector<>();
***REMOVED***
***REMOVED*** ***REMOVED*** add algorithms and dependencies to the worker
***REMOVED*** for (Algorithm a : as.getAlgorithms()) {
***REMOVED*** for (Role r : this.generateRoles(a)) {
***REMOVED*** algoRoles.add(r);
***REMOVED*** worker.addRole(r);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***to comment the for in case of just install algo
***REMOVED*** if(test){
***REMOVED*** for (Dependency d : a.getDependencies()) {
***REMOVED*** for (Role r : this.generateRoles(d)) {
***REMOVED*** worker.addRole(r);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED*** add static roles
***REMOVED*** for(Role r:this.getStaticRoleManager().getStaticRoles()) {
***REMOVED*** worker.addRole(r);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED*** generate the inventory
***REMOVED*** Inventory inventory = new Inventory();
***REMOVED*** for (Host h : cluster.getHosts()) {
***REMOVED*** AnsibleHost ah = new AnsibleHost(h.getName());
***REMOVED*** inventory.addHost(ah, "universe");
***REMOVED*** inventory.addHost(ah, "d4science");
***REMOVED*** ***REMOVED***
***REMOVED*** worker.setInventory(inventory);
***REMOVED***
***REMOVED*** ***REMOVED*** generate the playbook
***REMOVED*** Playbook playbook = new Playbook();
***REMOVED*** if(test){
***REMOVED*** playbook.setRemote_user("root");***REMOVED***
***REMOVED*** playbook.setRemote_user("gcube");
***REMOVED*** playbook.applyTo("universe");
***REMOVED*** for(Role r:algoRoles) {
***REMOVED*** ***REMOVED*** add only 'add' roles
***REMOVED*** if(!r.getName().endsWith("remove")) {
***REMOVED*** playbook.addRole(r.getName());
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** worker.setPlaybook(playbook);
***REMOVED***
***REMOVED*** ***REMOVED*** execute and save log locally
***REMOVED*** ***REMOVED***PrintStream console = System.out;
***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
***REMOVED*** path.mkdirs();
***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
***REMOVED*** FileOutputStream fos = new FileOutputStream(n);
***REMOVED*** PrintStream ps = new PrintStream(fos);
***REMOVED***
***REMOVED*** ***REMOVED***System.setErr(console);
***REMOVED***
***REMOVED*** worker.apply(as,ps,test);
***REMOVED*** ***REMOVED***System.setOut(console);
***REMOVED*** ***REMOVED***worker.apply();
***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED***
***REMOVED*** ***REMOVED*** destroy the worker
***REMOVED*** worker.destroy();
***REMOVED*** return worker;
***REMOVED*** ***REMOVED***

206
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -172,70 +172,70 @@ public class DataminerPoolManager {
return bufferScript;
***REMOVED***
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake");
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
new Thread(new Runnable() {
@Override
public void run() {
***REMOVED*** TODO Auto-generated method stub
try {
try {
addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
***REMOVED*** catch (SVNException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED*** catch (InterruptedException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED*** this line will execute immediately, not waiting for task to
***REMOVED*** complete
System.out.println(uuid);
return uuid;
***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
***REMOVED*** ***REMOVED*** create a fake algorithm set
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
***REMOVED*** algoSet.setName("fake");
***REMOVED*** algoSet.addAlgorithm(algorithm);
***REMOVED*** final String uuid = UUID.randomUUID().toString();
***REMOVED***
***REMOVED*** new Thread(new Runnable() {
***REMOVED*** @Override
***REMOVED*** public void run() {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** try {
***REMOVED*** try {
***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
***REMOVED*** ***REMOVED*** catch (SVNException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** catch (IOException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED*** catch (InterruptedException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***).start();
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for task to
***REMOVED*** ***REMOVED*** complete
***REMOVED*** System.out.println(uuid);
***REMOVED*** return uuid;
***REMOVED******REMOVED***
public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake");
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
new Thread(new Runnable() {
@Override
public void run() {
***REMOVED*** TODO Auto-generated method stub
try {
if(test){
addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED*** catch (InterruptedException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED*** catch (SVNException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED*** this line will execute immediately, not waiting for your task to
***REMOVED*** complete
System.out.println(uuid);
return uuid;
***REMOVED*** public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
***REMOVED*** ***REMOVED*** create a fake algorithm set
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
***REMOVED*** algoSet.setName("fake");
***REMOVED*** algoSet.addAlgorithm(algorithm);
***REMOVED*** final String uuid = UUID.randomUUID().toString();
***REMOVED***
***REMOVED*** new Thread(new Runnable() {
***REMOVED*** @Override
***REMOVED*** public void run() {
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** try {
***REMOVED*** if(test){
***REMOVED*** addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
***REMOVED*** ***REMOVED*** catch (IOException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED*** catch (InterruptedException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED*** catch (SVNException e) {
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
***REMOVED*** e.printStackTrace();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***).start();
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for your task to
***REMOVED*** ***REMOVED*** complete
***REMOVED*** System.out.println(uuid);
***REMOVED*** return uuid;
***REMOVED******REMOVED***
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
@ -249,52 +249,52 @@ public class DataminerPoolManager {
return new File(n.getPath()).toURI().toURL();
***REMOVED***
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
for (Host h : new HAProxy().listDataMinersByCluster()) {
***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) {
cluster.addHost(h);
***REMOVED***
***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
***REMOVED***
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
throws IOException, InterruptedException, SVNException {
***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
for (Host h : new HAProxy().listDataMinersByCluster()) {
if (h.getName().equals(hostname)) {
cluster.addHost(h);
***REMOVED***
***REMOVED***
***REMOVED*** if(ISClient.getHProxy().equals(hostname)){
***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname));
***REMOVED*** ***REMOVED***
***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
***REMOVED*** Cluster cluster = new Cluster();
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) {
***REMOVED*** ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) {
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED*** apply the changes
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
***REMOVED*** throws IOException, InterruptedException, SVNException {
***REMOVED***
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
***REMOVED*** Cluster cluster = new Cluster();
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) {
***REMOVED*** if (h.getName().equals(hostname)) {
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** if(ISClient.getHProxy().equals(hostname)){
***REMOVED*** ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname));
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** apply the changes
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED***
***REMOVED******REMOVED***
public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
throws IOException, InterruptedException, SVNException {
Cluster cluster = new Cluster();
Host h = new Host();
h.setName(hostname);
cluster.addHost(h);
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED***
***REMOVED*** public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
***REMOVED*** throws IOException, InterruptedException, SVNException {
***REMOVED*** Cluster cluster = new Cluster();
***REMOVED*** Host h = new Host();
***REMOVED*** h.setName(hostname);
***REMOVED*** cluster.addHost(h);
***REMOVED***
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** 2017 March 29

39
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java
View File

@ -23,29 +23,22 @@ public class AlgorithmBuilder {
if(category != null){
algo.setCategory(category);
***REMOVED***
***REMOVED***FIXME: do the same done for category
if (algo.getAlgorithmType() == null) {
algo.setAlgorithmType(algorithmType);
***REMOVED*** else
algo.setAlgorithmType(algo.getCategory());
if (algo.getSkipJava() == null) {
algo.setSkipJava(skipJava);
***REMOVED*** else
algo.setSkipJava(algo.getSkipJava());
if (algo.getName() == null) {
algo.setName(name);
***REMOVED*** else
algo.setName(algo.getName());
if (algo.getDescription() == null) {
algo.setDescription(description);
;
***REMOVED*** else
algo.setDescription(algo.getDescription());
if(algorithmType != null){
algo.setAlgorithmType(algorithmType);
***REMOVED***
if(skipJava != null){
algo.setSkipJava(skipJava);
***REMOVED***
if(skipJava != null){
algo.setSkipJava(skipJava);
***REMOVED***
if(name != null){
algo.setName(name);
***REMOVED***
if(description != null){
algo.setDescription(description);
***REMOVED***
return algo;
***REMOVED***

2
src/test/java/DataminerPoolManagerTest.java
View File

@ -220,7 +220,7 @@ public class DataminerPoolManagerTest {
ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble);
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
***REMOVED***

2
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java
View File

@ -220,7 +220,7 @@ public class DataminerPoolManagerTest {
ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble);
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false);
***REMOVED***