2017-03-03 19:10:02 +01:00
|
|
|
package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
|
|
|
|
|
|
|
***REMOVED***
|
2017-03-29 16:06:46 +02:00
|
|
|
import org.gcube.common.resources.gcore.GenericResource;
|
|
|
|
import org.gcube.common.resources.gcore.Resources;
|
|
|
|
***REMOVED***
|
2017-05-09 18:38:15 +02:00
|
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
|
2017-03-03 19:10:02 +01:00
|
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
2017-04-19 16:59:36 +02:00
|
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
|
2017-05-11 12:58:38 +02:00
|
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
2017-05-09 18:38:15 +02:00
|
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
|
2017-05-11 17:42:53 +02:00
|
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.util.Props;
|
2017-05-09 18:46:52 +02:00
|
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
2017-03-29 16:06:46 +02:00
|
|
|
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
|
|
|
|
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
|
|
|
import org.gcube.informationsystem.publisher.ScopedPublisher;
|
|
|
|
import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException;
|
|
|
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
|
|
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
2017-05-09 18:38:15 +02:00
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
2017-03-29 16:06:46 +02:00
|
|
|
|
2017-05-09 18:38:15 +02:00
|
|
|
import java.io.*;
|
2017-03-03 19:10:02 +01:00
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
2017-05-09 18:38:15 +02:00
|
|
|
import java.net.URLConnection;
|
2017-04-05 17:14:18 +02:00
|
|
|
***REMOVED***
|
2017-05-09 18:38:15 +02:00
|
|
|
import java.util.*;
|
2017-03-03 19:10:02 +01:00
|
|
|
|
2017-05-09 18:38:15 +02:00
|
|
|
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
|
|
|
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
|
|
|
|
|
|
|
public class DataminerPoolManager ***REMOVED***
|
2017-03-03 19:10:02 +01:00
|
|
|
|
|
|
|
|
2017-05-09 18:38:15 +02:00
|
|
|
|
2017-05-09 18:46:52 +02:00
|
|
|
private SVNUpdater svnUpdater;
|
|
|
|
|
|
|
|
public DataminerPoolManager()***REMOVED***
|
|
|
|
try ***REMOVED***
|
2017-05-11 18:08:17 +02:00
|
|
|
***REMOVED***TODO: read this from configuration - fatto
|
|
|
|
this.svnUpdater = new SVNUpdater(new Props().getSVNrepo());
|
2017-05-09 18:46:52 +02:00
|
|
|
***REMOVED*** catch (SVNException e) ***REMOVED***
|
|
|
|
e.printStackTrace();
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
|
|
|
|
|
|
|
|
|
2017-05-09 18:38:15 +02:00
|
|
|
***REMOVED***Algorithm algo) throws IOException, InterruptedException ***REMOVED***
|
|
|
|
|
2017-05-10 11:04:04 +02:00
|
|
|
|
|
|
|
Cluster cluster = getStagingDataminerCluster();
|
2017-05-09 18:38:15 +02:00
|
|
|
|
|
|
|
***REMOVED***Assumes the service is running in RPrototypingLab
|
|
|
|
String token = SecurityTokenProvider.instance.get();
|
|
|
|
|
|
|
|
return addAlgorithmToCluster(algo, cluster, true, "root", true, token);
|
|
|
|
***REMOVED***
|
|
|
|
|
|
|
|
***REMOVED***Algorithm algo, String targetVREToken) throws IOException, InterruptedException ***REMOVED***
|
|
|
|
Cluster cluster = new Cluster();
|
2017-05-11 18:08:17 +02:00
|
|
|
SecurityTokenProvider.instance.set(targetVREToken);
|
2017-05-09 18:38:15 +02:00
|
|
|
for (Host h : new HAProxy().listDataMinersByCluster()) ***REMOVED***
|
|
|
|
cluster.addHost(h);
|
|
|
|
***REMOVED***
|
|
|
|
return addAlgorithmToCluster(algo, cluster, false, "gcube", false, targetVREToken);
|
|
|
|
|
|
|
|
***REMOVED***
|
|
|
|
|
2017-05-11 12:58:38 +02:00
|
|
|
***REMOVED***1. to complete
|
2017-05-10 11:04:04 +02:00
|
|
|
private Cluster getStagingDataminerCluster()***REMOVED***
|
|
|
|
Cluster cluster = new Cluster();
|
|
|
|
Host h = new Host();
|
2017-05-11 17:42:53 +02:00
|
|
|
Props p = new Props();
|
2017-05-11 12:58:38 +02:00
|
|
|
***REMOVED***TODO: read this from configuration or IS?
|
2017-05-11 17:42:53 +02:00
|
|
|
h.setName(p.getStagingHost());
|
2017-05-10 11:04:04 +02:00
|
|
|
cluster.addHost(h);
|
|
|
|
return cluster;
|
2017-05-09 18:38:15 +02:00
|
|
|
***REMOVED***
|
2017-05-11 12:58:38 +02:00
|
|
|
|
|
|
|
|
|
|
|
private void createISResource(Algorithm algo, String vreToken) ***REMOVED***
|
|
|
|
ISClient client = new ISClient();
|
|
|
|
for (Algorithm a : client.getAlgoFromIs()) ***REMOVED***
|
|
|
|
if (a.getName().equals(algo.getName())) ***REMOVED***
|
|
|
|
continue;
|
|
|
|
***REMOVED***
|
|
|
|
if (!a.getName().equals(algo.getName()))***REMOVED***
|
|
|
|
new ISClient().addAlgToIs(algo, vreToken);
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED*** TODO: create the resource only if not already present
|
|
|
|
***REMOVED***
|
|
|
|
|
2017-05-09 18:38:15 +02:00
|
|
|
|
|
|
|
private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED***
|
|
|
|
for (Dependency d : algo.getDependencies()) ***REMOVED***
|
|
|
|
|
|
|
|
if (d.getType().equals("os")) ***REMOVED***
|
|
|
|
List<String> ls = new LinkedList<String>();
|
|
|
|
ls.add(d.getName());
|
2017-05-09 18:46:52 +02:00
|
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
|
2017-05-09 18:38:15 +02:00
|
|
|
***REMOVED***
|
|
|
|
if (d.getType().equals("cran")) ***REMOVED***
|
|
|
|
List<String> ls = new LinkedList<String>();
|
|
|
|
ls.add(d.getName());
|
2017-05-09 18:46:52 +02:00
|
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
|
2017-05-09 18:38:15 +02:00
|
|
|
***REMOVED***
|
|
|
|
if (d.getType().equals("github")) ***REMOVED***
|
|
|
|
List<String> ls = new LinkedList<String>();
|
|
|
|
ls.add(d.getName());
|
2017-05-09 18:46:52 +02:00
|
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
|
2017-05-09 18:38:15 +02:00
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
|
|
|
|
|
|
|
|
***REMOVED****
|
|
|
|
*
|
|
|
|
* @param algo
|
|
|
|
* @param dataminerCluster
|
|
|
|
* @return uuid of the execution
|
|
|
|
***REMOVED***
|
|
|
|
public String addAlgorithmToCluster(
|
|
|
|
final Algorithm algo,
|
|
|
|
Cluster dataminerCluster,
|
|
|
|
boolean includeAlgorithmDependencies,
|
|
|
|
String user,
|
|
|
|
final boolean stagingVRE,
|
|
|
|
final String targetVREToken) throws IOException ***REMOVED***
|
|
|
|
|
|
|
|
AnsibleBridge ansibleBridge = new AnsibleBridge();
|
|
|
|
final AnsibleWorker worker = ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user);
|
|
|
|
|
|
|
|
new Thread(new Runnable() ***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
public void run() ***REMOVED***
|
|
|
|
try ***REMOVED***
|
|
|
|
File path = new File(worker.getWorkdir() + File.separator + "logs");
|
|
|
|
path.mkdirs();
|
|
|
|
File n = new File(path + File.separator + worker.getWorkerId());
|
|
|
|
FileOutputStream fos = new FileOutputStream(n);
|
|
|
|
PrintStream ps = new PrintStream(fos);
|
|
|
|
int retValue = worker.execute(ps);
|
|
|
|
System.out.println("Log stored to to " + n.getAbsolutePath());
|
|
|
|
|
|
|
|
if(retValue == 0) ***REMOVED***
|
|
|
|
updateSVNDependencies(algo, stagingVRE);
|
2017-05-11 17:03:59 +02:00
|
|
|
***REMOVED***createISResource(algo, targetVREToken);
|
2017-05-09 18:38:15 +02:00
|
|
|
***REMOVED***
|
|
|
|
***REMOVED*** destroy the worker
|
|
|
|
worker.destroy();
|
|
|
|
***REMOVED*** catch (Exception e) ***REMOVED***
|
|
|
|
e.printStackTrace();
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***).start();
|
|
|
|
|
|
|
|
***REMOVED*** this line will execute immediately, not waiting for task to
|
|
|
|
***REMOVED*** complete
|
|
|
|
return worker.getWorkerId();
|
|
|
|
***REMOVED***
|
|
|
|
|
|
|
|
|
2017-05-09 17:10:03 +02:00
|
|
|
|
2017-03-29 16:06:46 +02:00
|
|
|
public String getScriptFromURL(URL url) throws IOException ***REMOVED***
|
|
|
|
if (url == null) ***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
URLConnection yc = url.openConnection();
|
|
|
|
BufferedReader input = new BufferedReader(new InputStreamReader(yc.getInputStream()));
|
|
|
|
String line;
|
|
|
|
StringBuffer buffer = new StringBuffer();
|
|
|
|
while ((line = input.readLine()) != null) ***REMOVED***
|
|
|
|
buffer.append(line + "\n");
|
|
|
|
***REMOVED***
|
|
|
|
String bufferScript = buffer.substring(0, buffer.length());
|
|
|
|
input.close();
|
|
|
|
return bufferScript;
|
2017-03-03 19:10:02 +01:00
|
|
|
***REMOVED***
|
2017-03-29 16:06:46 +02:00
|
|
|
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** create a fake algorithm set
|
|
|
|
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
|
|
|
|
***REMOVED*** algoSet.setName("fake");
|
|
|
|
***REMOVED*** algoSet.addAlgorithm(algorithm);
|
|
|
|
***REMOVED*** final String uuid = UUID.randomUUID().toString();
|
2017-03-03 19:10:02 +01:00
|
|
|
***REMOVED***
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED*** new Thread(new Runnable() ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** public void run() ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** try ***REMOVED***
|
|
|
|
***REMOVED*** try ***REMOVED***
|
|
|
|
***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
|
|
|
|
***REMOVED*** ***REMOVED*** catch (SVNException e) ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
|
|
***REMOVED*** e.printStackTrace();
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
|
|
***REMOVED*** e.printStackTrace();
|
|
|
|
***REMOVED*** ***REMOVED*** catch (InterruptedException e) ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
|
|
***REMOVED*** e.printStackTrace();
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***).start();
|
|
|
|
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for task to
|
|
|
|
***REMOVED*** ***REMOVED*** complete
|
|
|
|
***REMOVED*** System.out.println(uuid);
|
|
|
|
***REMOVED*** return uuid;
|
|
|
|
***REMOVED******REMOVED***
|
|
|
|
|
|
|
|
***REMOVED*** public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** create a fake algorithm set
|
|
|
|
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
|
|
|
|
***REMOVED*** algoSet.setName("fake");
|
|
|
|
***REMOVED*** algoSet.addAlgorithm(algorithm);
|
|
|
|
***REMOVED*** final String uuid = UUID.randomUUID().toString();
|
2017-03-03 19:10:02 +01:00
|
|
|
***REMOVED***
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED*** new Thread(new Runnable() ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** public void run() ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** try ***REMOVED***
|
|
|
|
***REMOVED*** if(test)***REMOVED***
|
|
|
|
***REMOVED*** addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
|
|
***REMOVED*** e.printStackTrace();
|
|
|
|
***REMOVED*** ***REMOVED*** catch (InterruptedException e) ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
|
|
***REMOVED*** e.printStackTrace();
|
|
|
|
***REMOVED*** ***REMOVED*** catch (SVNException e) ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
|
|
***REMOVED*** e.printStackTrace();
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***).start();
|
|
|
|
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for your task to
|
|
|
|
***REMOVED*** ***REMOVED*** complete
|
|
|
|
***REMOVED*** System.out.println(uuid);
|
|
|
|
***REMOVED*** return uuid;
|
|
|
|
***REMOVED******REMOVED***
|
2017-03-29 16:06:46 +02:00
|
|
|
|
2017-05-09 18:38:15 +02:00
|
|
|
|
2017-03-29 16:06:46 +02:00
|
|
|
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException ***REMOVED***
|
|
|
|
|
2017-04-26 17:39:24 +02:00
|
|
|
File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/"
|
2017-03-29 16:06:46 +02:00
|
|
|
+ a + File.separator + "logs");
|
|
|
|
path.mkdirs();
|
|
|
|
File n = new File(path + File.separator + a);
|
|
|
|
***REMOVED*** String addr = InetAddress.getLocalHost().getHostAddress();
|
|
|
|
|
|
|
|
return new File(n.getPath()).toURI().toURL();
|
2017-03-03 19:10:02 +01:00
|
|
|
***REMOVED***
|
2017-03-29 16:06:46 +02:00
|
|
|
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException ***REMOVED***
|
2017-03-03 19:10:02 +01:00
|
|
|
***REMOVED***
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
|
|
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
|
|
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) ***REMOVED***
|
|
|
|
***REMOVED*** cluster.addHost(h);
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** apply the changes
|
|
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED******REMOVED***
|
2017-03-29 16:06:46 +02:00
|
|
|
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED*** public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
|
|
|
***REMOVED*** throws IOException, InterruptedException, SVNException ***REMOVED***
|
2017-03-03 19:10:02 +01:00
|
|
|
***REMOVED***
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
|
|
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
|
|
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) ***REMOVED***
|
|
|
|
***REMOVED*** if (h.getName().equals(hostname)) ***REMOVED***
|
|
|
|
***REMOVED*** cluster.addHost(h);
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** if(ISClient.getHProxy().equals(hostname))***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname));
|
|
|
|
***REMOVED*** ***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** ***REMOVED*** apply the changes
|
|
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED******REMOVED***
|
2017-04-06 11:58:51 +02:00
|
|
|
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
|
|
|
***REMOVED*** throws IOException, InterruptedException, SVNException ***REMOVED***
|
|
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
|
|
***REMOVED*** Host h = new Host();
|
|
|
|
***REMOVED*** h.setName(hostname);
|
|
|
|
***REMOVED*** cluster.addHost(h);
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
2017-05-09 17:10:03 +02:00
|
|
|
***REMOVED***
|
2017-05-10 11:31:28 +02:00
|
|
|
***REMOVED******REMOVED***
|
2017-04-06 10:59:46 +02:00
|
|
|
|
2017-03-03 19:10:02 +01:00
|
|
|
|
2017-03-29 16:06:46 +02:00
|
|
|
***REMOVED***
|