569 lines
16 KiB
Java
569 lines
16 KiB
Java
|
//package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
||
|
//
|
||
|
//import java.io.BufferedReader;
|
||
|
//import java.io.File;
|
||
|
//import java.io.IOException;
|
||
|
//import java.io.InputStreamReader;
|
||
|
//import java.net.MalformedURLException;
|
||
|
//import java.net.URL;
|
||
|
//import java.net.URLConnection;
|
||
|
//import java.net.UnknownHostException;
|
||
|
//import java.util.UUID;
|
||
|
//
|
||
|
//import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||
|
//import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
||
|
//import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||
|
//import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
|
||
|
//import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||
|
//import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
|
||
|
//import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
|
||
|
//import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
|
||
|
//import org.slf4j.Logger;
|
||
|
//import org.slf4j.LoggerFactory;
|
||
|
//
|
||
|
//public class DataminerPoolManager implements PoolManager {
|
||
|
//
|
||
|
//
|
||
|
// private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
||
|
//
|
||
|
//
|
||
|
//// static Collection<Algorithm> algorithms;
|
||
|
////
|
||
|
//// static Collection<AlgorithmSet> sets;
|
||
|
////
|
||
|
//// static {
|
||
|
//// algorithms = new Vector<>();
|
||
|
//// }
|
||
|
////
|
||
|
//// public DataminerPoolManager() {
|
||
|
//// }
|
||
|
////
|
||
|
//// /**
|
||
|
//// * Add a new algorithm to the set of known ones. No further action is expected
|
||
|
//// * on the pool.
|
||
|
//// */
|
||
|
//// public void publishAlgorithm(Algorithm algorithm) {
|
||
|
//// algorithms.add(algorithm);
|
||
|
//// }
|
||
|
////
|
||
|
//// /**
|
||
|
//// * Re-deploy the given algorithm wherever it's installed
|
||
|
//// *
|
||
|
//// * @param algorithm
|
||
|
//// */
|
||
|
//// /*
|
||
|
//// * public void updateAlgorithm(Algorithm algorithm) { // TODO implement this }
|
||
|
//// */
|
||
|
////
|
||
|
//// /**
|
||
|
//// * Add the give algorithm to the given set
|
||
|
//// *
|
||
|
//// * @param algorithmId
|
||
|
//// * @param setId
|
||
|
//// */
|
||
|
//// public void addAlgorithmToSet(String algorithmName, String setName) {
|
||
|
//// AlgorithmSet set = this.getAlgorithmSet(setName);
|
||
|
//// Algorithm algorithm = this.getAlgorithm(algorithmName);
|
||
|
//// if (set != null && algorithm != null) {
|
||
|
//// set.addAlgorithm(algorithm);
|
||
|
//// this.updateClusters();
|
||
|
//// }
|
||
|
//// }
|
||
|
////
|
||
|
//// /**
|
||
|
//// * Apply the given set of algorithms to the given cluster
|
||
|
//// *
|
||
|
//// * @param setId
|
||
|
//// * @param clusterId
|
||
|
//// */
|
||
|
//// public void applyAlgorithmSetToCluster(String setName, String clusterName) {
|
||
|
//// AlgorithmSet set = this.getAlgorithmSet(setName);
|
||
|
//// Cluster cluster = new ISClient().getCluster(clusterName);
|
||
|
//// if (set != null && cluster != null) {
|
||
|
//// cluster.addAlgorithmSet(set);
|
||
|
//// this.updateClusters();
|
||
|
//// }
|
||
|
//// }
|
||
|
////
|
||
|
//// private AlgorithmSet getAlgorithmSet(String name) {
|
||
|
//// for (AlgorithmSet set : sets) {
|
||
|
//// if (name.equals(set.getName())) {
|
||
|
//// return set;
|
||
|
//// }
|
||
|
//// }
|
||
|
//// return null;
|
||
|
//// }
|
||
|
////
|
||
|
//// private Algorithm getAlgorithm(String name) {
|
||
|
//// for (Algorithm a : algorithms) {
|
||
|
//// if (name.equals(a.getName())) {
|
||
|
//// return a;
|
||
|
//// }
|
||
|
//// }
|
||
|
//// return null;
|
||
|
//// }
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
////
|
||
|
//// public void getLogId(final Algorithm algorithm, final String vre) {
|
||
|
//// new Thread() {
|
||
|
//// public void run() {
|
||
|
//// while (true) {
|
||
|
//// try {
|
||
|
//// addAlgorithmToVRE(algorithm, vre);
|
||
|
//// } catch (Exception e) {
|
||
|
//// //log here
|
||
|
//// }
|
||
|
//// }
|
||
|
//// }
|
||
|
//// }.start();
|
||
|
//// }
|
||
|
////
|
||
|
////
|
||
|
//
|
||
|
//// public String getLogId(){
|
||
|
//// PrintStream console = System.out;
|
||
|
//// File path = new File(worker.getWorkdir() + File.separator + "logs");
|
||
|
//// path.mkdirs();
|
||
|
//// File n = new File(path + File.separator + worker.getWorkerId());
|
||
|
//// FileOutputStream fos = new FileOutputStream(n);
|
||
|
//// PrintStream ps = new PrintStream(fos);
|
||
|
//// System.setOut(ps);
|
||
|
//// worker.apply();
|
||
|
//// System.setOut(console);
|
||
|
//// worker.apply();
|
||
|
//// System.out.println("Log stored to to " + n.getAbsolutePath());
|
||
|
//// }
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
//// public String getLogById(String id) throws IOException {
|
||
|
//// String strLine = null;
|
||
|
//// try{
|
||
|
//// FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
|
||
|
//// BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
|
||
|
//// /* read log line by line */
|
||
|
//// while ((strLine = br.readLine()) != null) {
|
||
|
//// /* parse strLine to obtain what you want */
|
||
|
//// System.out.println (strLine);
|
||
|
//// }
|
||
|
//// br.close();
|
||
|
//// } catch (Exception e) {
|
||
|
//// System.err.println("Error: " + e.getMessage());
|
||
|
//// }
|
||
|
//// return strLine;
|
||
|
//// }
|
||
|
//
|
||
|
//
|
||
|
// public String getScriptFromURL(URL url) throws IOException {
|
||
|
// if (url == null) {
|
||
|
// return null;
|
||
|
// }
|
||
|
// URLConnection yc = url.openConnection();
|
||
|
// BufferedReader input = new BufferedReader(new InputStreamReader(
|
||
|
// yc.getInputStream()));
|
||
|
// String line;
|
||
|
// StringBuffer buffer = new StringBuffer();
|
||
|
// while ((line = input.readLine()) != null) {
|
||
|
// buffer.append(line + "\n");
|
||
|
// }
|
||
|
// String bufferScript = buffer.substring(0, buffer.length());
|
||
|
// input.close();
|
||
|
// return bufferScript;
|
||
|
// }
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
// /**
|
||
|
// * Publish the given algorithm in the given VRE
|
||
|
// *
|
||
|
// * @param algorithmName
|
||
|
// * @param vre
|
||
|
// *
|
||
|
// */
|
||
|
// public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException {
|
||
|
// // create a fake algorithm set
|
||
|
// final AlgorithmSet algoSet = new AlgorithmSet();
|
||
|
// algoSet.setName("fake");
|
||
|
// algoSet.addAlgorithm(algorithm);
|
||
|
// final String uuid = UUID.randomUUID().toString();
|
||
|
//
|
||
|
// new Thread(new Runnable() {
|
||
|
// @Override
|
||
|
// public void run() {
|
||
|
// // TODO Auto-generated method stub
|
||
|
// try {
|
||
|
// addAlgorithmsToVRE(algoSet, vre, uuid);
|
||
|
// } catch (IOException e) {
|
||
|
// // TODO Auto-generated catch block
|
||
|
// e.printStackTrace();
|
||
|
// }
|
||
|
// }
|
||
|
// }).start();
|
||
|
// //this line will execute immediately, not waiting for your task to complete
|
||
|
// System.out.println(uuid);
|
||
|
// return uuid;
|
||
|
// }
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
// public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{
|
||
|
//
|
||
|
// File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs");
|
||
|
// path.mkdirs();
|
||
|
// File n = new File(path + File.separator +a);
|
||
|
// //String addr = InetAddress.getLocalHost().getHostAddress();
|
||
|
//
|
||
|
// return new File(n.getPath()).toURI().toURL();
|
||
|
// }
|
||
|
//
|
||
|
//
|
||
|
// public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException {
|
||
|
//
|
||
|
// // create the cluster (dataminers in the vre)
|
||
|
// Cluster cluster = new Cluster();
|
||
|
// for(Host h:new ISClient().listDataminersInVRE()) {
|
||
|
// cluster.addHost(h);
|
||
|
// }
|
||
|
//
|
||
|
// // apply the changes
|
||
|
// AnsibleBridge a = new AnsibleBridge();
|
||
|
// return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId();
|
||
|
//
|
||
|
// }
|
||
|
//
|
||
|
// public Algorithm extractAlgorithm(String url) throws IOException {
|
||
|
// return new AlgorithmPackageParser().parsePackage(url);
|
||
|
// }
|
||
|
//
|
||
|
//
|
||
|
//@Override
|
||
|
//public void getLogId(Algorithm algo, String vre) {
|
||
|
// // TODO Auto-generated method stub
|
||
|
//
|
||
|
//}
|
||
|
//
|
||
|
//
|
||
|
//@Override
|
||
|
//public String getLogById(String logId) throws IOException {
|
||
|
// // TODO Auto-generated method stub
|
||
|
// return null;
|
||
|
//}
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
//
|
||
|
//}
|
||
|
package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
||
|
|
||
|
import java.io.BufferedReader;
|
||
|
import java.io.File;
|
||
|
import java.io.IOException;
|
||
|
import java.io.InputStreamReader;
|
||
|
import java.net.MalformedURLException;
|
||
|
import java.net.URL;
|
||
|
import java.net.URLConnection;
|
||
|
import java.net.UnknownHostException;
|
||
|
import java.util.UUID;
|
||
|
|
||
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||
|
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
||
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
|
||
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
|
||
|
import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
|
||
|
import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
|
||
|
import org.slf4j.Logger;
|
||
|
import org.slf4j.LoggerFactory;
|
||
|
|
||
|
public class DataminerPoolManager implements PoolManager {
|
||
|
|
||
|
|
||
|
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
||
|
|
||
|
|
||
|
// static Collection<Algorithm> algorithms;
|
||
|
//
|
||
|
// static Collection<AlgorithmSet> sets;
|
||
|
//
|
||
|
// static {
|
||
|
// algorithms = new Vector<>();
|
||
|
// }
|
||
|
//
|
||
|
// public DataminerPoolManager() {
|
||
|
// }
|
||
|
//
|
||
|
// /**
|
||
|
// * Add a new algorithm to the set of known ones. No further action is expected
|
||
|
// * on the pool.
|
||
|
// */
|
||
|
// public void publishAlgorithm(Algorithm algorithm) {
|
||
|
// algorithms.add(algorithm);
|
||
|
// }
|
||
|
//
|
||
|
// /**
|
||
|
// * Re-deploy the given algorithm wherever it's installed
|
||
|
// *
|
||
|
// * @param algorithm
|
||
|
// */
|
||
|
// /*
|
||
|
// * public void updateAlgorithm(Algorithm algorithm) { // TODO implement this }
|
||
|
// */
|
||
|
//
|
||
|
// /**
|
||
|
// * Add the give algorithm to the given set
|
||
|
// *
|
||
|
// * @param algorithmId
|
||
|
// * @param setId
|
||
|
// */
|
||
|
// public void addAlgorithmToSet(String algorithmName, String setName) {
|
||
|
// AlgorithmSet set = this.getAlgorithmSet(setName);
|
||
|
// Algorithm algorithm = this.getAlgorithm(algorithmName);
|
||
|
// if (set != null && algorithm != null) {
|
||
|
// set.addAlgorithm(algorithm);
|
||
|
// this.updateClusters();
|
||
|
// }
|
||
|
// }
|
||
|
//
|
||
|
// /**
|
||
|
// * Apply the given set of algorithms to the given cluster
|
||
|
// *
|
||
|
// * @param setId
|
||
|
// * @param clusterId
|
||
|
// */
|
||
|
// public void applyAlgorithmSetToCluster(String setName, String clusterName) {
|
||
|
// AlgorithmSet set = this.getAlgorithmSet(setName);
|
||
|
// Cluster cluster = new ISClient().getCluster(clusterName);
|
||
|
// if (set != null && cluster != null) {
|
||
|
// cluster.addAlgorithmSet(set);
|
||
|
// this.updateClusters();
|
||
|
// }
|
||
|
// }
|
||
|
//
|
||
|
// private AlgorithmSet getAlgorithmSet(String name) {
|
||
|
// for (AlgorithmSet set : sets) {
|
||
|
// if (name.equals(set.getName())) {
|
||
|
// return set;
|
||
|
// }
|
||
|
// }
|
||
|
// return null;
|
||
|
// }
|
||
|
//
|
||
|
// private Algorithm getAlgorithm(String name) {
|
||
|
// for (Algorithm a : algorithms) {
|
||
|
// if (name.equals(a.getName())) {
|
||
|
// return a;
|
||
|
// }
|
||
|
// }
|
||
|
// return null;
|
||
|
// }
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
//
|
||
|
// public void getLogId(final Algorithm algorithm, final String vre) {
|
||
|
// new Thread() {
|
||
|
// public void run() {
|
||
|
// while (true) {
|
||
|
// try {
|
||
|
// addAlgorithmToVRE(algorithm, vre);
|
||
|
// } catch (Exception e) {
|
||
|
// //log here
|
||
|
// }
|
||
|
// }
|
||
|
// }
|
||
|
// }.start();
|
||
|
// }
|
||
|
//
|
||
|
//
|
||
|
|
||
|
// public String getLogId(){
|
||
|
// PrintStream console = System.out;
|
||
|
// File path = new File(worker.getWorkdir() + File.separator + "logs");
|
||
|
// path.mkdirs();
|
||
|
// File n = new File(path + File.separator + worker.getWorkerId());
|
||
|
// FileOutputStream fos = new FileOutputStream(n);
|
||
|
// PrintStream ps = new PrintStream(fos);
|
||
|
// System.setOut(ps);
|
||
|
// worker.apply();
|
||
|
// System.setOut(console);
|
||
|
// worker.apply();
|
||
|
// System.out.println("Log stored to to " + n.getAbsolutePath());
|
||
|
// }
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
// public String getLogById(String id) throws IOException {
|
||
|
// String strLine = null;
|
||
|
// try{
|
||
|
// FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
|
||
|
// BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
|
||
|
// /* read log line by line */
|
||
|
// while ((strLine = br.readLine()) != null) {
|
||
|
// /* parse strLine to obtain what you want */
|
||
|
// System.out.println (strLine);
|
||
|
// }
|
||
|
// br.close();
|
||
|
// } catch (Exception e) {
|
||
|
// System.err.println("Error: " + e.getMessage());
|
||
|
// }
|
||
|
// return strLine;
|
||
|
// }
|
||
|
|
||
|
|
||
|
public String getScriptFromURL(URL url) throws IOException {
|
||
|
if (url == null) {
|
||
|
return null;
|
||
|
}
|
||
|
URLConnection yc = url.openConnection();
|
||
|
BufferedReader input = new BufferedReader(new InputStreamReader(
|
||
|
yc.getInputStream()));
|
||
|
String line;
|
||
|
StringBuffer buffer = new StringBuffer();
|
||
|
while ((line = input.readLine()) != null) {
|
||
|
buffer.append(line + "\n");
|
||
|
}
|
||
|
String bufferScript = buffer.substring(0, buffer.length());
|
||
|
input.close();
|
||
|
return bufferScript;
|
||
|
}
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
/**
|
||
|
* Publish the given algorithm in the given VRE
|
||
|
*
|
||
|
* @param algorithmName
|
||
|
* @param vre
|
||
|
*
|
||
|
*/
|
||
|
public String addAlgorithmToVRE(Algorithm algorithm, final String vre)
|
||
|
throws IOException {
|
||
|
// create a fake algorithm set
|
||
|
final AlgorithmSet algoSet = new AlgorithmSet();
|
||
|
algoSet.setName("fake");
|
||
|
algoSet.addAlgorithm(algorithm);
|
||
|
final String uuid = UUID.randomUUID().toString();
|
||
|
|
||
|
new Thread(new Runnable() {
|
||
|
@Override
|
||
|
public void run() {
|
||
|
// TODO Auto-generated method stub
|
||
|
try {
|
||
|
addAlgorithmsToVRE(algoSet, vre, uuid);
|
||
|
} catch (IOException e) {
|
||
|
// TODO Auto-generated catch block
|
||
|
e.printStackTrace();
|
||
|
}
|
||
|
}
|
||
|
}).start();
|
||
|
// this line will execute immediately, not waiting for your task to complete
|
||
|
System.out.println(uuid);
|
||
|
return uuid;
|
||
|
}
|
||
|
|
||
|
public String addAlgorithmToHost(Algorithm algorithm, final String hostname)
|
||
|
throws IOException {
|
||
|
// create a fake algorithm set
|
||
|
final AlgorithmSet algoSet = new AlgorithmSet();
|
||
|
algoSet.setName("fake");
|
||
|
algoSet.addAlgorithm(algorithm);
|
||
|
final String uuid = UUID.randomUUID().toString();
|
||
|
|
||
|
new Thread(new Runnable() {
|
||
|
@Override
|
||
|
public void run() {
|
||
|
// TODO Auto-generated method stub
|
||
|
try {
|
||
|
addAlgorithmsToHost(algoSet, hostname, uuid);
|
||
|
} catch (IOException e) {
|
||
|
// TODO Auto-generated catch block
|
||
|
e.printStackTrace();
|
||
|
}
|
||
|
}
|
||
|
}).start();
|
||
|
// this line will execute immediately, not waiting for your task to complete
|
||
|
System.out.println(uuid);
|
||
|
return uuid;
|
||
|
}
|
||
|
|
||
|
|
||
|
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{
|
||
|
|
||
|
File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs");
|
||
|
path.mkdirs();
|
||
|
File n = new File(path + File.separator +a);
|
||
|
//String addr = InetAddress.getLocalHost().getHostAddress();
|
||
|
|
||
|
return new File(n.getPath()).toURI().toURL();
|
||
|
}
|
||
|
|
||
|
|
||
|
|
||
|
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException {
|
||
|
|
||
|
// create the cluster (dataminers in the vre)
|
||
|
Cluster cluster = new Cluster();
|
||
|
for(Host h:new ISClient().listDataminersInVRE()) {
|
||
|
cluster.addHost(h);
|
||
|
}
|
||
|
|
||
|
// apply the changes
|
||
|
AnsibleBridge a = new AnsibleBridge();
|
||
|
return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId();
|
||
|
|
||
|
}
|
||
|
|
||
|
public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid) throws IOException {
|
||
|
|
||
|
// create the cluster (dataminers in the vre)
|
||
|
Cluster cluster = new Cluster();
|
||
|
cluster.addHost(new ISClient().getDataminer(hostname));
|
||
|
// for(Host h:new ISClient().listDataminersInVRE()) {
|
||
|
// cluster.addHost(h);
|
||
|
// }
|
||
|
|
||
|
// apply the changes
|
||
|
AnsibleBridge a = new AnsibleBridge();
|
||
|
return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId();
|
||
|
|
||
|
}
|
||
|
|
||
|
|
||
|
public Algorithm extractAlgorithm(String url) throws IOException {
|
||
|
return new AlgorithmPackageParser().parsePackage(url);
|
||
|
}
|
||
|
|
||
|
|
||
|
@Override
|
||
|
public void getLogId(Algorithm algo, String vre) {
|
||
|
// TODO Auto-generated method stub
|
||
|
|
||
|
}
|
||
|
|
||
|
|
||
|
@Override
|
||
|
public String getLogById(String logId) throws IOException {
|
||
|
// TODO Auto-generated method stub
|
||
|
return null;
|
||
|
}
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
|
||
|
}
|
||
|
|
||
|
|