310 lines
12 KiB
Java
310 lines
12 KiB
Java
package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
|
|
|
***REMOVED***
|
|
import org.gcube.common.resources.gcore.GenericResource;
|
|
import org.gcube.common.resources.gcore.Resources;
|
|
***REMOVED***
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.util.Props;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
|
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
|
|
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
|
import org.gcube.informationsystem.publisher.ScopedPublisher;
|
|
import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException;
|
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
|
***REMOVED***
|
|
***REMOVED***
|
|
import org.tmatesoft.svn.core.SVNException;
|
|
|
|
import java.io.*;
|
|
import java.net.MalformedURLException;
|
|
***REMOVED***
|
|
import java.net.URLConnection;
|
|
import java.net.UnknownHostException;
|
|
import java.util.*;
|
|
|
|
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
|
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
|
|
|
public class DataminerPoolManager {
|
|
|
|
|
|
|
|
private SVNUpdater svnUpdater;
|
|
|
|
public DataminerPoolManager(){
|
|
try {
|
|
***REMOVED***TODO: read this from configuration - fatto
|
|
this.svnUpdater = new SVNUpdater(new Props().getSVNrepo());
|
|
***REMOVED*** catch (SVNException e) {
|
|
e.printStackTrace();
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
|
|
|
|
public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
|
|
|
|
|
|
Cluster cluster = getStagingDataminerCluster();
|
|
|
|
***REMOVED***Assumes the service is running in RPrototypingLab
|
|
String token = SecurityTokenProvider.instance.get();
|
|
|
|
return addAlgorithmToCluster(algo, cluster, true, "root", true, token);
|
|
***REMOVED***
|
|
|
|
public String publishAlgorithm(Algorithm algo, String targetVREToken) throws IOException, InterruptedException {
|
|
Cluster cluster = new Cluster();
|
|
***REMOVED***
|
|
for (Host h : new HAProxy().listDataMinersByCluster()) {
|
|
cluster.addHost(h);
|
|
***REMOVED***
|
|
return addAlgorithmToCluster(algo, cluster, false, "gcube", false, targetVREToken);
|
|
|
|
***REMOVED***
|
|
|
|
***REMOVED***1. to complete
|
|
private Cluster getStagingDataminerCluster(){
|
|
Cluster cluster = new Cluster();
|
|
Host h = new Host();
|
|
Props p = new Props();
|
|
***REMOVED***TODO: read this from configuration or IS?
|
|
h.setName(p.getStagingHost());
|
|
cluster.addHost(h);
|
|
return cluster;
|
|
***REMOVED***
|
|
|
|
|
|
private void createISResource(Algorithm algo, String vreToken) {
|
|
ISClient client = new ISClient();
|
|
for (Algorithm a : client.getAlgoFromIs()) {
|
|
if (a.getName().equals(algo.getName())) {
|
|
continue;
|
|
***REMOVED***
|
|
if (!a.getName().equals(algo.getName())){
|
|
new ISClient().addAlgToIs(algo, vreToken);
|
|
***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED*** TODO: create the resource only if not already present
|
|
***REMOVED***
|
|
|
|
|
|
private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
|
|
for (Dependency d : algo.getDependencies()) {
|
|
|
|
if (d.getType().equals("os")) {
|
|
List<String> ls = new LinkedList<String>();
|
|
ls.add(d.getName());
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
|
|
***REMOVED***
|
|
if (d.getType().equals("cran")) {
|
|
List<String> ls = new LinkedList<String>();
|
|
ls.add(d.getName());
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
|
|
***REMOVED***
|
|
if (d.getType().equals("github")) {
|
|
List<String> ls = new LinkedList<String>();
|
|
ls.add(d.getName());
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
|
|
***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
|
|
/**
|
|
*
|
|
* @param algo
|
|
* @param dataminerCluster
|
|
* @return uuid of the execution
|
|
*/
|
|
public String addAlgorithmToCluster(
|
|
final Algorithm algo,
|
|
Cluster dataminerCluster,
|
|
boolean includeAlgorithmDependencies,
|
|
String user,
|
|
final boolean stagingVRE,
|
|
final String targetVREToken) throws IOException {
|
|
|
|
AnsibleBridge ansibleBridge = new AnsibleBridge();
|
|
final AnsibleWorker worker = ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user);
|
|
|
|
new Thread(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
try {
|
|
File path = new File(worker.getWorkdir() + File.separator + "logs");
|
|
path.mkdirs();
|
|
File n = new File(path + File.separator + worker.getWorkerId());
|
|
FileOutputStream fos = new FileOutputStream(n);
|
|
PrintStream ps = new PrintStream(fos);
|
|
int retValue = worker.execute(ps);
|
|
System.out.println("Log stored to to " + n.getAbsolutePath());
|
|
|
|
if(retValue == 0) {
|
|
updateSVNDependencies(algo, stagingVRE);
|
|
***REMOVED***createISResource(algo, targetVREToken);
|
|
***REMOVED***
|
|
***REMOVED*** destroy the worker
|
|
worker.destroy();
|
|
***REMOVED*** catch (Exception e) {
|
|
e.printStackTrace();
|
|
***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED***).start();
|
|
|
|
***REMOVED*** this line will execute immediately, not waiting for task to
|
|
***REMOVED*** complete
|
|
return worker.getWorkerId();
|
|
***REMOVED***
|
|
|
|
|
|
|
|
public String getScriptFromURL(URL url) throws IOException {
|
|
if (url == null) {
|
|
return null;
|
|
***REMOVED***
|
|
URLConnection yc = url.openConnection();
|
|
BufferedReader input = new BufferedReader(new InputStreamReader(yc.getInputStream()));
|
|
String line;
|
|
StringBuffer buffer = new StringBuffer();
|
|
while ((line = input.readLine()) != null) {
|
|
buffer.append(line + "\n");
|
|
***REMOVED***
|
|
String bufferScript = buffer.substring(0, buffer.length());
|
|
input.close();
|
|
return bufferScript;
|
|
***REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
|
***REMOVED*** ***REMOVED*** create a fake algorithm set
|
|
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
|
|
***REMOVED*** algoSet.setName("fake");
|
|
***REMOVED*** algoSet.addAlgorithm(algorithm);
|
|
***REMOVED*** final String uuid = UUID.randomUUID().toString();
|
|
***REMOVED***
|
|
***REMOVED*** new Thread(new Runnable() {
|
|
***REMOVED*** @Override
|
|
***REMOVED*** public void run() {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
|
|
***REMOVED*** try {
|
|
***REMOVED*** try {
|
|
***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
|
|
***REMOVED*** ***REMOVED*** catch (SVNException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** catch (IOException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED*** catch (InterruptedException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***).start();
|
|
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for task to
|
|
***REMOVED*** ***REMOVED*** complete
|
|
***REMOVED*** System.out.println(uuid);
|
|
***REMOVED*** return uuid;
|
|
***REMOVED******REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
|
***REMOVED*** ***REMOVED*** create a fake algorithm set
|
|
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
|
|
***REMOVED*** algoSet.setName("fake");
|
|
***REMOVED*** algoSet.addAlgorithm(algorithm);
|
|
***REMOVED*** final String uuid = UUID.randomUUID().toString();
|
|
***REMOVED***
|
|
***REMOVED*** new Thread(new Runnable() {
|
|
***REMOVED*** @Override
|
|
***REMOVED*** public void run() {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
|
|
***REMOVED*** try {
|
|
***REMOVED*** if(test){
|
|
***REMOVED*** addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
|
|
***REMOVED*** ***REMOVED*** catch (IOException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED*** catch (InterruptedException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED*** catch (SVNException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***).start();
|
|
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for your task to
|
|
***REMOVED*** ***REMOVED*** complete
|
|
***REMOVED*** System.out.println(uuid);
|
|
***REMOVED*** return uuid;
|
|
***REMOVED******REMOVED***
|
|
|
|
|
|
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
|
|
|
|
File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/"
|
|
+ a + File.separator + "logs");
|
|
path.mkdirs();
|
|
File n = new File(path + File.separator + a);
|
|
***REMOVED*** String addr = InetAddress.getLocalHost().getHostAddress();
|
|
|
|
return new File(n.getPath()).toURI().toURL();
|
|
***REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) {
|
|
***REMOVED*** ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) {
|
|
***REMOVED*** cluster.addHost(h);
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED*** apply the changes
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
|
|
***REMOVED***
|
|
***REMOVED******REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
|
***REMOVED*** throws IOException, InterruptedException, SVNException {
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) {
|
|
***REMOVED*** if (h.getName().equals(hostname)) {
|
|
***REMOVED*** cluster.addHost(h);
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** if(ISClient.getHProxy().equals(hostname)){
|
|
***REMOVED*** ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname));
|
|
***REMOVED*** ***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** apply the changes
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
|
***REMOVED***
|
|
***REMOVED******REMOVED***
|
|
|
|
***REMOVED***
|
|
***REMOVED*** public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
|
***REMOVED*** throws IOException, InterruptedException, SVNException {
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
***REMOVED*** Host h = new Host();
|
|
***REMOVED*** h.setName(hostname);
|
|
***REMOVED*** cluster.addHost(h);
|
|
***REMOVED***
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
|
***REMOVED***
|
|
***REMOVED******REMOVED***
|
|
|
|
|
|
***REMOVED***
|