406 lines
16 KiB
Java
406 lines
16 KiB
Java
package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
|
|
|
***REMOVED***
|
|
import org.gcube.common.resources.gcore.GenericResource;
|
|
import org.gcube.common.resources.gcore.Resources;
|
|
***REMOVED***
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
|
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
|
|
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
|
import org.gcube.informationsystem.publisher.ScopedPublisher;
|
|
import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException;
|
|
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
|
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
|
***REMOVED***
|
|
***REMOVED***
|
|
import org.tmatesoft.svn.core.SVNException;
|
|
|
|
import java.io.*;
|
|
import java.net.MalformedURLException;
|
|
***REMOVED***
|
|
import java.net.URLConnection;
|
|
import java.net.UnknownHostException;
|
|
import java.util.*;
|
|
|
|
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
|
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
|
|
|
public class DataminerPoolManager {
|
|
|
|
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
|
|
|
|
|
private SVNUpdater svnUpdater;
|
|
|
|
public DataminerPoolManager(){
|
|
try {
|
|
***REMOVED***TODO: read this from configuration
|
|
this.svnUpdater = new SVNUpdater(
|
|
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
|
***REMOVED*** catch (SVNException e) {
|
|
e.printStackTrace();
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
|
|
|
|
public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
|
|
|
|
|
|
Cluster cluster = getStagingDataminerCluster();
|
|
|
|
***REMOVED***Assumes the service is running in RPrototypingLab
|
|
String token = SecurityTokenProvider.instance.get();
|
|
|
|
return addAlgorithmToCluster(algo, cluster, true, "root", true, token);
|
|
***REMOVED***
|
|
|
|
public String publishAlgorithm(Algorithm algo, String targetVREToken) throws IOException, InterruptedException {
|
|
Cluster cluster = new Cluster();
|
|
for (Host h : new HAProxy().listDataMinersByCluster()) {
|
|
cluster.addHost(h);
|
|
***REMOVED***
|
|
return addAlgorithmToCluster(algo, cluster, false, "gcube", false, targetVREToken);
|
|
|
|
***REMOVED***
|
|
|
|
|
|
private Cluster getStagingDataminerCluster(){
|
|
Cluster cluster = new Cluster();
|
|
Host h = new Host();
|
|
***REMOVED***TODO: read this from configuration
|
|
h.setName("test");
|
|
cluster.addHost(h);
|
|
return cluster;
|
|
***REMOVED***
|
|
|
|
private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
|
|
for (Dependency d : algo.getDependencies()) {
|
|
|
|
if (d.getType().equals("os")) {
|
|
List<String> ls = new LinkedList<String>();
|
|
ls.add(d.getName());
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
|
|
***REMOVED***
|
|
if (d.getType().equals("cran")) {
|
|
List<String> ls = new LinkedList<String>();
|
|
ls.add(d.getName());
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
|
|
***REMOVED***
|
|
if (d.getType().equals("github")) {
|
|
List<String> ls = new LinkedList<String>();
|
|
ls.add(d.getName());
|
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
|
|
***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
|
|
private void createISResource(Algorithm algo, String vreToken){
|
|
***REMOVED***TODO: implement method
|
|
|
|
***REMOVED***TODO: create the resource only if not already present
|
|
***REMOVED***
|
|
|
|
|
|
|
|
/**
|
|
*
|
|
* @param algo
|
|
* @param dataminerCluster
|
|
* @return uuid of the execution
|
|
*/
|
|
public String addAlgorithmToCluster(
|
|
final Algorithm algo,
|
|
Cluster dataminerCluster,
|
|
boolean includeAlgorithmDependencies,
|
|
String user,
|
|
final boolean stagingVRE,
|
|
final String targetVREToken) throws IOException {
|
|
|
|
AnsibleBridge ansibleBridge = new AnsibleBridge();
|
|
final AnsibleWorker worker = ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user);
|
|
|
|
new Thread(new Runnable() {
|
|
@Override
|
|
public void run() {
|
|
try {
|
|
File path = new File(worker.getWorkdir() + File.separator + "logs");
|
|
path.mkdirs();
|
|
File n = new File(path + File.separator + worker.getWorkerId());
|
|
FileOutputStream fos = new FileOutputStream(n);
|
|
PrintStream ps = new PrintStream(fos);
|
|
int retValue = worker.execute(ps);
|
|
System.out.println("Log stored to to " + n.getAbsolutePath());
|
|
|
|
if(retValue == 0) {
|
|
updateSVNDependencies(algo, stagingVRE);
|
|
createISResource(algo, targetVREToken);
|
|
***REMOVED***
|
|
***REMOVED*** destroy the worker
|
|
worker.destroy();
|
|
***REMOVED*** catch (Exception e) {
|
|
e.printStackTrace();
|
|
***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED***).start();
|
|
|
|
***REMOVED*** this line will execute immediately, not waiting for task to
|
|
***REMOVED*** complete
|
|
return worker.getWorkerId();
|
|
***REMOVED***
|
|
|
|
|
|
|
|
public String getScriptFromURL(URL url) throws IOException {
|
|
if (url == null) {
|
|
return null;
|
|
***REMOVED***
|
|
URLConnection yc = url.openConnection();
|
|
BufferedReader input = new BufferedReader(new InputStreamReader(yc.getInputStream()));
|
|
String line;
|
|
StringBuffer buffer = new StringBuffer();
|
|
while ((line = input.readLine()) != null) {
|
|
buffer.append(line + "\n");
|
|
***REMOVED***
|
|
String bufferScript = buffer.substring(0, buffer.length());
|
|
input.close();
|
|
return bufferScript;
|
|
***REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
|
***REMOVED*** ***REMOVED*** create a fake algorithm set
|
|
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
|
|
***REMOVED*** algoSet.setName("fake");
|
|
***REMOVED*** algoSet.addAlgorithm(algorithm);
|
|
***REMOVED*** final String uuid = UUID.randomUUID().toString();
|
|
***REMOVED***
|
|
***REMOVED*** new Thread(new Runnable() {
|
|
***REMOVED*** @Override
|
|
***REMOVED*** public void run() {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
|
|
***REMOVED*** try {
|
|
***REMOVED*** try {
|
|
***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
|
|
***REMOVED*** ***REMOVED*** catch (SVNException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** catch (IOException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED*** catch (InterruptedException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***).start();
|
|
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for task to
|
|
***REMOVED*** ***REMOVED*** complete
|
|
***REMOVED*** System.out.println(uuid);
|
|
***REMOVED*** return uuid;
|
|
***REMOVED******REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
|
***REMOVED*** ***REMOVED*** create a fake algorithm set
|
|
***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet();
|
|
***REMOVED*** algoSet.setName("fake");
|
|
***REMOVED*** algoSet.addAlgorithm(algorithm);
|
|
***REMOVED*** final String uuid = UUID.randomUUID().toString();
|
|
***REMOVED***
|
|
***REMOVED*** new Thread(new Runnable() {
|
|
***REMOVED*** @Override
|
|
***REMOVED*** public void run() {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
|
|
***REMOVED*** try {
|
|
***REMOVED*** if(test){
|
|
***REMOVED*** addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
|
|
***REMOVED*** ***REMOVED*** catch (IOException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED*** catch (InterruptedException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED*** catch (SVNException e) {
|
|
***REMOVED*** ***REMOVED*** TODO Auto-generated catch block
|
|
***REMOVED*** e.printStackTrace();
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***).start();
|
|
***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for your task to
|
|
***REMOVED*** ***REMOVED*** complete
|
|
***REMOVED*** System.out.println(uuid);
|
|
***REMOVED*** return uuid;
|
|
***REMOVED******REMOVED***
|
|
|
|
|
|
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
|
|
|
|
File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/"
|
|
+ a + File.separator + "logs");
|
|
path.mkdirs();
|
|
File n = new File(path + File.separator + a);
|
|
***REMOVED*** String addr = InetAddress.getLocalHost().getHostAddress();
|
|
|
|
return new File(n.getPath()).toURI().toURL();
|
|
***REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) {
|
|
***REMOVED*** ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) {
|
|
***REMOVED*** cluster.addHost(h);
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED*** apply the changes
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
|
|
***REMOVED***
|
|
***REMOVED******REMOVED***
|
|
|
|
***REMOVED*** public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
|
***REMOVED*** throws IOException, InterruptedException, SVNException {
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre)
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) {
|
|
***REMOVED*** if (h.getName().equals(hostname)) {
|
|
***REMOVED*** cluster.addHost(h);
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** if(ISClient.getHProxy().equals(hostname)){
|
|
***REMOVED*** ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname));
|
|
***REMOVED*** ***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** apply the changes
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
|
***REMOVED***
|
|
***REMOVED******REMOVED***
|
|
|
|
***REMOVED***
|
|
***REMOVED*** public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
|
|
***REMOVED*** throws IOException, InterruptedException, SVNException {
|
|
***REMOVED*** Cluster cluster = new Cluster();
|
|
***REMOVED*** Host h = new Host();
|
|
***REMOVED*** h.setName(hostname);
|
|
***REMOVED*** cluster.addHost(h);
|
|
***REMOVED***
|
|
***REMOVED*** AnsibleBridge a = new AnsibleBridge();
|
|
***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
|
|
***REMOVED***
|
|
***REMOVED******REMOVED***
|
|
|
|
|
|
***REMOVED*** 2017 March 29
|
|
|
|
public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception {
|
|
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
|
AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher);
|
|
String id = resource.id();
|
|
LOGGER.debug("Trying to remove {***REMOVED*** with ID {***REMOVED*** from {***REMOVED***", resource.getClass().getSimpleName(), id,
|
|
ScopeProvider.instance.get());
|
|
***REMOVED*** scopedPublisher.remove(resource, scopes);
|
|
advancedScopedPublisher.forceRemove(resource);
|
|
LOGGER.debug("{***REMOVED*** with ID {***REMOVED*** removed successfully", resource.getClass().getSimpleName(), id);
|
|
***REMOVED***
|
|
|
|
public void publishScopedResource(GenericResource a, List<String> scopes)
|
|
throws RegistryNotFoundException, Exception {
|
|
StringWriter stringWriter = new StringWriter();
|
|
Resources.marshal(a, stringWriter);
|
|
|
|
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
|
try {
|
|
System.out.println(scopes);
|
|
System.out.println(stringWriter);
|
|
scopedPublisher.create(a, scopes);
|
|
***REMOVED*** catch (RegistryNotFoundException e) {
|
|
System.out.println(e);
|
|
throw e;
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
public void addAlgToIs(Algorithm algo) {
|
|
GenericResource a = new GenericResource();
|
|
a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
|
|
a.profile().newBody(this.getAlgoBody(algo));
|
|
try {
|
|
publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() ***REMOVED***));
|
|
***REMOVED*** catch (Exception e) {
|
|
e.printStackTrace();
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
private String getAlgoBody(Algorithm algo) {
|
|
return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
|
|
+ "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
|
|
+ algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
|
|
+ "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
|
|
***REMOVED***
|
|
|
|
public void updateAlg(Algorithm algo) {
|
|
|
|
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
|
|
|
SimpleQuery query = queryFor(GenericResource.class);
|
|
query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource");
|
|
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
|
|
List<GenericResource> ds = client.submit(query);
|
|
if (ds.isEmpty()) {
|
|
return;
|
|
***REMOVED***
|
|
GenericResource a = ds.get(0);
|
|
a.profile().newBody(this.getAlgoBody(algo));
|
|
try {
|
|
scopedPublisher.update(a);
|
|
***REMOVED*** catch (RegistryNotFoundException e) {
|
|
e.printStackTrace();
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
private Algorithm convertAlgo(GenericResource a) {
|
|
Algorithm out = new Algorithm();
|
|
|
|
***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
|
|
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
|
|
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
|
|
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
|
|
out.setName(a.profile().name());
|
|
out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent());
|
|
out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent());
|
|
out.setDescription(a.profile().description());
|
|
|
|
Set<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency> deps = new HashSet<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency>();
|
|
for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) {
|
|
org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency();
|
|
d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent());
|
|
deps.add(d1);
|
|
***REMOVED***
|
|
out.setDependencies(deps);
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
public Set<Algorithm> getAlgoFromIs() {
|
|
***REMOVED*** TODO Auto-generated method stub
|
|
|
|
Set<Algorithm> out = new HashSet<Algorithm>();
|
|
SimpleQuery query = queryFor(GenericResource.class);
|
|
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
|
|
.setResult("$resource");
|
|
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
|
|
List<GenericResource> ds = client.submit(query);
|
|
for (GenericResource a : ds) {
|
|
out.add(this.convertAlgo(a));
|
|
***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
|
|
***REMOVED***
|