diff --git a/.classpath b/.classpath
deleted file mode 100644
index 70eaec6..0000000
--- a/.classpath
+++ /dev/null
@@ -1,34 +0,0 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/.project b/.project
deleted file mode 100644
index 367d621..0000000
--- a/.project
+++ /dev/null
@@ -1,42 +0,0 @@
-
-
- dataminer-pool-manager
-
-
-
-
-
- org.eclipse.wst.jsdt.core.javascriptValidator
-
-
-
-
- org.eclipse.wst.common.project.facet.core.builder
-
-
-
-
- org.eclipse.jdt.core.javabuilder
-
-
-
-
- org.eclipse.wst.validation.validationbuilder
-
-
-
-
- org.eclipse.m2e.core.maven2Builder
-
-
-
-
-
- org.eclipse.jem.workbench.JavaEMFNature
- org.eclipse.wst.common.modulecore.ModuleCoreNature
- org.eclipse.jdt.core.javanature
- org.eclipse.m2e.core.maven2Nature
- org.eclipse.wst.common.project.facet.core.nature
- org.eclipse.wst.jsdt.core.jsNature
-
-
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
index 2b6660a..efc0677 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
@@ -116,9 +116,32 @@ public class AnsibleWorker {
***REMOVED*** Serialize role in the workdir
AnsibleSerializeHelper.serializeRole(r, this.getRolesDir());
***REMOVED***
-
-
-
+
+
+
+ public int execute(PrintStream ps)
+ throws IOException, InterruptedException, SVNException {
+
+ System.out.println(this.getWorkdir());
+ try {
+ Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath()
+ + " " + this.getPlaybookFile().getAbsolutePath());
+
+ inheritIO(p.getInputStream(), ps);
+ inheritIO(p.getErrorStream(), ps);
+
+
+ return p.waitFor();
+
+ ***REMOVED*** catch (IOException e) {
+ e.printStackTrace();
+ ***REMOVED***
+
+ return -1;
+***REMOVED***
+
+
+
public void apply(AlgorithmSet as, PrintStream ps, boolean test)
throws IOException, InterruptedException, SVNException {
***REMOVED*** TODO execute the playbook and return output
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
index b5a4096..00cdc26 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
@@ -89,9 +89,55 @@ public class AnsibleBridge {
- public AnsibleWorker createWorker() {
+ public AnsibleWorker createWorker(Algorithm algorithm, Cluster dataminerCluster, boolean includeAlgorithmDependencies, String user) throws IOException {
File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString());
AnsibleWorker worker = new AnsibleWorker(workerRoot);
+
+ List algoRoles = new Vector<>();
+
+ ***REMOVED*** add algorithms and dependencies to the worker
+ for (Role r : this.generateRoles(algorithm)) {
+ algoRoles.add(r);
+ worker.addRole(r);
+ ***REMOVED***
+
+ ***REMOVED***to comment the for in case of just install algo
+ if(includeAlgorithmDependencies){
+ for (Dependency d : algorithm.getDependencies()) {
+ for (Role r : this.generateRoles(d)) {
+ worker.addRole(r);
+ ***REMOVED***
+ ***REMOVED***
+ ***REMOVED***
+
+
+ ***REMOVED*** add static roles
+ for(Role r:this.getStaticRoleManager().getStaticRoles()) {
+ worker.addRole(r);
+ ***REMOVED***
+
+ ***REMOVED*** generate the inventory
+ Inventory inventory = new Inventory();
+ for (Host h : dataminerCluster.getHosts()) {
+ AnsibleHost ah = new AnsibleHost(h.getName());
+ inventory.addHost(ah, "universe");
+ inventory.addHost(ah, "d4science");
+ ***REMOVED***
+ worker.setInventory(inventory);
+
+ ***REMOVED*** generate the playbook
+ Playbook playbook = new Playbook();
+ playbook.setRemote_user(user);
+ playbook.applyTo("universe");
+ for(Role r:algoRoles) {
+ ***REMOVED*** add only 'add' roles
+ if(!r.getName().endsWith("remove")) {
+ playbook.addRole(r.getName());
+ ***REMOVED***
+ ***REMOVED***
+
+ worker.setPlaybook(playbook);
+
return worker;
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
index 60a2c6b..600a3be 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
@@ -42,7 +42,12 @@ public interface PoolManager {
String addAlgorithmToVRE(Algorithm algo, String vre, boolean test ) throws IOException, InterruptedException;
String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException;
-
+
+
+ String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException;
+ String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException;
+
+
Algorithm extractAlgorithm(String url) throws IOException;
String getLogById(String logId) throws IOException;
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
index fbd63a2..e8a00b9 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
@@ -1,281 +1,52 @@
-***REMOVED***package org.gcube.dataanalysis.dataminer.poolmanager.rest;
-***REMOVED***
-***REMOVED******REMOVED***
-***REMOVED***import java.net.MalformedURLException;
-***REMOVED***import java.net.ProxySelector;
-***REMOVED******REMOVED***
-***REMOVED***import java.net.UnknownHostException;
-***REMOVED***
-***REMOVED***import javax.ws.rs.GET;
-***REMOVED***import javax.ws.rs.Path;
-***REMOVED***import javax.ws.rs.Produces;
-***REMOVED***import javax.ws.rs.QueryParam;
-***REMOVED***
-***REMOVED******REMOVED***
-***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
-***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
-***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
-***REMOVED******REMOVED***
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***@Path("/")
-***REMOVED***public class RestPoolManager implements PoolManager {
-***REMOVED***
-***REMOVED*** private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class);
-***REMOVED***
-***REMOVED*** private PoolManager service = new DataminerPoolManager();
-***REMOVED***
-***REMOVED***
-***REMOVED*** @GET
-***REMOVED*** @Path("/add")
-***REMOVED*** @Produces("text/plain")
-***REMOVED*** public String addAlgorithmToVRE(
-***REMOVED*** @QueryParam("algorithm") String algorithm,
-***REMOVED*** @QueryParam("vre") String vre,
-***REMOVED*** @QueryParam("name") String name,
-***REMOVED*** @QueryParam("description") String description,
-***REMOVED*** @QueryParam("category") String category,
-***REMOVED*** @QueryParam("algorithmType") String algorithmType,
-***REMOVED*** @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
-***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
-***REMOVED*** LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
-***REMOVED*** Algorithm algo = service.extractAlgorithm(algorithm);
-***REMOVED***
-***REMOVED*** if (algo.getCategory() == null){
-***REMOVED*** algo.setCategory(category);
-***REMOVED*** ***REMOVED*** else category = algo.getCategory();
-***REMOVED***
-***REMOVED*** if (algo.getAlgorithmType() == null){
-***REMOVED*** algo.setAlgorithmType(algorithmType);
-***REMOVED*** ***REMOVED*** else algorithmType = algo.getCategory();
-***REMOVED***
-***REMOVED*** if (algo.getSkipJava() == null){
-***REMOVED*** algo.setSkipJava(skipJava);
-***REMOVED*** ***REMOVED*** else skipJava = algo.getSkipJava();
-***REMOVED***
-***REMOVED*** if (algo.getName() == null){
-***REMOVED*** algo.setCategory(name);
-***REMOVED*** ***REMOVED*** else name = algo.getName();
-***REMOVED***
-***REMOVED*** if (algo.getDescription() == null){
-***REMOVED*** algo.setDescription(description);;
-***REMOVED*** ***REMOVED*** else description = algo.getDescription();
-***REMOVED***
-***REMOVED*** return service.addAlgorithmToVRE(algo, vre);
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED*** @GET
-***REMOVED*** @Path("/log")
-***REMOVED*** @Produces("text/plain")
-***REMOVED*** public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException {
-***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
-***REMOVED*** LOGGER.debug("Returning Log =" + logUrl);
-***REMOVED*** return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl));
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED*** @Override
-***REMOVED*** public Algorithm extractAlgorithm(String url) throws IOException {
-***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
-***REMOVED*** return null;
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED*** public static void main(String[] args) throws IOException, InterruptedException {
-***REMOVED*** RestPoolManager a = new RestPoolManager();
-***REMOVED*** ***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
-***REMOVED***
-***REMOVED***
-***REMOVED*** ***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
-***REMOVED***
-***REMOVED*** ScopeProvider.instance.set("/gcube/devNext/NextNext");
-***REMOVED*** a.addAlgorithmToVRE(
-***REMOVED*** "http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0",
-***REMOVED*** "/gcube/devNext/NextNext",
-***REMOVED*** null, null, "test", "transducerers", "N");
-***REMOVED***
-***REMOVED*** ***REMOVED***System.out.println(a.getLogById("34ac474d-b9df-4929-87e1-2a0ae26cf898"));
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED*** @Override
-***REMOVED*** public void getLogId(Algorithm algo, String vre) {
-***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
-***REMOVED***
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED*** @Override
-***REMOVED*** public String getScriptFromURL(URL logId) throws IOException {
-***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
-***REMOVED*** return null;
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED*** @Override
-***REMOVED*** public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException {
-***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
-***REMOVED*** return null;
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED*** @Override
-***REMOVED*** public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException {
-***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
-***REMOVED*** return null;
-***REMOVED******REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED***
-***REMOVED******REMOVED***
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
-import java.awt.color.ICC_ColorSpace;
-***REMOVED***
-import java.net.MalformedURLException;
-import java.net.ProxySelector;
-***REMOVED***
-import java.net.UnknownHostException;
***REMOVED***
***REMOVED***
-import java.util.Random;
-import java.util.Set;
-import java.util.UUID;
-
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.Path;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-
-import org.apache.commons.lang.math.RandomUtils;
-***REMOVED***
-***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
-import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNException;
-import ch.qos.logback.core.subst.Token;
+import javax.ws.rs.*;
+***REMOVED***
+import java.net.MalformedURLException;
+***REMOVED***
+import java.net.UnknownHostException;
+***REMOVED***
+import java.util.Set;
@Path("/")
public class RestPoolManager implements PoolManager {
private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class);
- private PoolManager service = new DataminerPoolManager();
+ private DataminerPoolManager service = new DataminerPoolManager();
+
+
+ @GET
+ @Path("/algorithm/stage")
+ @Produces("text/plain")
+ public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException {
+ Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
+ return this.service.stageAlgorithm(algo);
+***REMOVED***
+
+
+ @GET
+ @Path("/algorithm/add")
+ @Produces("text/plain")
+ public String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException {
+ Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
+ return this.service.publishAlgorithm(algo, targetVREToken);
+***REMOVED***
/*
* /scopes/ POST ***REMOVED*** add an algorithm to all dataminers in the scope
* /hosts/ POST ***REMOVED*** add an algorithm to the given host
*/
- @GET
- @Path("/scopes/add")
- @Produces("text/plain")
- public String addAlgorithmToVRE(
- @QueryParam("algorithm") String algorithm,
- @QueryParam("name") String name,
- @QueryParam("description") String description,
- @QueryParam("category") String category,
- @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
- @DefaultValue("N") @QueryParam("skipJava") String skipJava,
- @DefaultValue("false") @QueryParam("publish") boolean publish,
- @DefaultValue("false") @QueryParam("updateSVN") boolean updateSVN)
- throws IOException, InterruptedException, SVNException {
- Algorithm algo = this.getAlgorithm(algorithm, /*vre*/null, null, name, description, category, algorithmType, skipJava);
- ***REMOVED*** publish algo
- if (publish) {
- service.addAlgToIs(algo);
- ***REMOVED***
- return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),/*updateSVN,*/false);
-***REMOVED***
-
- ***REMOVED***Test Phase
- ***REMOVED***The DMPM contacts the Staging Dataminer and installs the algorithm and the dependencies
- @GET
- @Path("/algo/add")
- @Produces("text/plain")
- public String addAlgorithmToStagingHost(
- @QueryParam("algorithm") String algorithm,
- @DefaultValue("insert hostname here") @QueryParam("hostname") String hostname,
- @QueryParam("name") String name,
- @QueryParam("description") String description,
- @QueryParam("category") String category,
- @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
- @DefaultValue("N") @QueryParam("skipJava") String skipJava,
- @DefaultValue("prodtoken") @QueryParam("prodtoken") String prodtoken,
- @DefaultValue("true") @QueryParam("test") boolean test)
- throws IOException, InterruptedException, SVNException {
- String id="";
- Algorithm algo = this.getAlgorithm(algorithm, ScopeProvider.instance.get(), hostname, name, description, category, algorithmType,skipJava);
-
- ***REMOVED***test phase
- if(test){
- service.addAlgorithmToHost(algo, hostname,true);
- id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),true);
- ***REMOVED***
- ***REMOVED***release phase
- if(!test){
- SecurityTokenProvider.instance.set(prodtoken);
- id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),false);
- ***REMOVED***
-
- return id;
-
-***REMOVED***
-
-
- private Algorithm getAlgorithm(String algorithm, String vre, String hostname, String name, String description,
- String category, String algorithmType, String skipJava) throws IOException, InterruptedException {
- Algorithm algo = service.extractAlgorithm(algorithm);
-
- if (algo.getCategory() == null) {
- algo.setCategory(category);
- ***REMOVED*** else
- algo.setCategory(algo.getCategory());
-
- if (algo.getAlgorithmType() == null) {
- algo.setAlgorithmType(algorithmType);
- ***REMOVED*** else
- algo.setAlgorithmType(algo.getCategory());
-
- if (algo.getSkipJava() == null) {
- algo.setSkipJava(skipJava);
- ***REMOVED*** else
- algo.setSkipJava(algo.getSkipJava());
-
- if (algo.getName() == null) {
- algo.setName(name);
- ***REMOVED*** else
- algo.setName(algo.getName());
-
- if (algo.getDescription() == null) {
- algo.setDescription(description);
- ;
- ***REMOVED*** else
- algo.setDescription(algo.getDescription());
-
- return algo;
-***REMOVED***
-
@GET
@Path("/log")
@Produces("text/plain")
@@ -304,40 +75,6 @@ public class RestPoolManager implements PoolManager {
***REMOVED*** System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3"));
***REMOVED*** System.out.println(aa.getAlgoFromIs());
-
- RestPoolManager a = new RestPoolManager();
- a.addAlgorithmToVRE(
- "http:***REMOVED***data.d4science.org/TVc0TW9Ud1FjYlppK0NHd2pvU0owNmRFWHE4OW4xSGNHbWJQNStIS0N6Yz0",
- "test",
- null,
- null,
- "N",
- "transducerers",
- false,
- false);
-
-***REMOVED*** a.addAlgorithmToHost(
-***REMOVED*** "http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0",
-***REMOVED*** "dataminer1-pre.d4science.org",
-***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers",
-***REMOVED*** "N",false, false);
- RestPoolManager b = new RestPoolManager();
-
- b.addAlgorithmToStagingHost(
- "http:***REMOVED***data.d4science.org/TVc0TW9Ud1FjYlppK0NHd2pvU0owNmRFWHE4OW4xSGNHbWJQNStIS0N6Yz0",
- "dataminer1-devnext.d4science.org",
- null,
- null,
- "N",
- "transducerers",
- false,
- false
-***REMOVED*** PoolManager aa = new DataminerPoolManager();
-***REMOVED*** List ls = new LinkedList();
-***REMOVED*** String afa = "test";
-***REMOVED*** ls.add(afa);
-***REMOVED***
-***REMOVED*** System.out.println(aa.updateSVN("r_deb_pkgs.txt", ls));
***REMOVED***
@Override
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
index c5d3e58..cc895b0 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
@@ -1,68 +1,142 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service;
-import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
-import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
-
***REMOVED***
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-***REMOVED***
-***REMOVED***
-import java.io.StringWriter;
-import java.net.MalformedURLException;
-***REMOVED***
-import java.net.URLConnection;
-import java.net.UnknownHostException;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.HashSet;
-***REMOVED***
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.UUID;
-
import org.gcube.common.resources.gcore.GenericResource;
-import org.gcube.common.resources.gcore.HostingNode;
import org.gcube.common.resources.gcore.Resources;
-import org.gcube.common.resources.gcore.Software.Profile.Dependency;
***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
-import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
-***REMOVED***
-***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
-import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
import org.gcube.informationsystem.publisher.ScopedPublisher;
import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
-
***REMOVED***
***REMOVED***
-import org.tmatesoft.svn.core.SVNCommitInfo;
import org.tmatesoft.svn.core.SVNException;
-import org.tmatesoft.svn.core.SVNURL;
-import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
-import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
-import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
-import org.tmatesoft.svn.core.io.ISVNEditor;
-import org.tmatesoft.svn.core.io.SVNRepository;
-import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
-import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
-import org.tmatesoft.svn.core.wc.SVNWCUtil;
-public class DataminerPoolManager implements PoolManager {
+import java.io.*;
+import java.net.MalformedURLException;
+***REMOVED***
+import java.net.URLConnection;
+import java.net.UnknownHostException;
+import java.util.*;
+
+import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
+import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
+
+public class DataminerPoolManager {
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
+
+ public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
+
+ Cluster cluster = new Cluster();
+ Host h = new Host();
+ h.setName(getStagingDataminerHostname());
+ cluster.addHost(h);
+
+ ***REMOVED***Assumes the service is running in RPrototypingLab
+ String token = SecurityTokenProvider.instance.get();
+
+ return addAlgorithmToCluster(algo, cluster, true, "root", true, token);
+***REMOVED***
+
+ public String publishAlgorithm(Algorithm algo, String targetVREToken) throws IOException, InterruptedException {
+ Cluster cluster = new Cluster();
+ for (Host h : new HAProxy().listDataMinersByCluster()) {
+ cluster.addHost(h);
+ ***REMOVED***
+ return addAlgorithmToCluster(algo, cluster, false, "gcube", false, targetVREToken);
+
+***REMOVED***
+
+
+ private String getStagingDataminerHostname(){
+ return null;
+***REMOVED***
+
+ private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
+ for (Dependency d : algo.getDependencies()) {
+
+ if (d.getType().equals("os")) {
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ this.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
+ ***REMOVED***
+ if (d.getType().equals("cran")) {
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ this.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
+ ***REMOVED***
+ if (d.getType().equals("github")) {
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ this.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
+ ***REMOVED***
+ ***REMOVED***
+***REMOVED***
+
+
+ private void createISResource(Algorithm algo, String vreToken){
+ ***REMOVED***TODO: implement method
+
+ ***REMOVED***TODO: create the resource only if not already present
+***REMOVED***
+
+
+
+ /**
+ *
+ * @param algo
+ * @param dataminerCluster
+ * @return uuid of the execution
+ */
+ public String addAlgorithmToCluster(
+ final Algorithm algo,
+ Cluster dataminerCluster,
+ boolean includeAlgorithmDependencies,
+ String user,
+ final boolean stagingVRE,
+ final String targetVREToken) throws IOException {
+
+ AnsibleBridge ansibleBridge = new AnsibleBridge();
+ final AnsibleWorker worker = ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user);
+
+ new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ File path = new File(worker.getWorkdir() + File.separator + "logs");
+ path.mkdirs();
+ File n = new File(path + File.separator + worker.getWorkerId());
+ FileOutputStream fos = new FileOutputStream(n);
+ PrintStream ps = new PrintStream(fos);
+ int retValue = worker.execute(ps);
+ System.out.println("Log stored to to " + n.getAbsolutePath());
+
+ if(retValue == 0) {
+ updateSVNDependencies(algo, stagingVRE);
+ createISResource(algo, targetVREToken);
+ ***REMOVED***
+ ***REMOVED*** destroy the worker
+ worker.destroy();
+ ***REMOVED*** catch (Exception e) {
+ e.printStackTrace();
+ ***REMOVED***
+ ***REMOVED***
+ ***REMOVED***).start();
+
+ ***REMOVED*** this line will execute immediately, not waiting for task to
+ ***REMOVED*** complete
+ return worker.getWorkerId();
+***REMOVED***
+
+
public String getScriptFromURL(URL url) throws IOException {
if (url == null) {
@@ -152,6 +226,7 @@ public class DataminerPoolManager implements PoolManager {
return uuid;
***REMOVED***
+
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/"
@@ -209,23 +284,7 @@ public class DataminerPoolManager implements PoolManager {
return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
***REMOVED***
-
- public Algorithm extractAlgorithm(String url) throws IOException {
- return new AlgorithmPackageParser().parsePackage(url);
-***REMOVED***
-
- @Override
- public void getLogId(Algorithm algo, String vre) {
- ***REMOVED*** TODO Auto-generated method stub
-
-***REMOVED***
-
- @Override
- public String getLogById(String logId) throws IOException {
- ***REMOVED*** TODO Auto-generated method stub
- return null;
-***REMOVED***
***REMOVED*** 2017 March 29
@@ -256,7 +315,6 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED***
***REMOVED***
- @Override
public void addAlgToIs(Algorithm algo) {
GenericResource a = new GenericResource();
a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
@@ -317,16 +375,6 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED***
***REMOVED***
- ***REMOVED*** public Algorithm getAlgoById(String id) {
- ***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) {
- ***REMOVED*** if (aa.getId().equals(id)) {
- ***REMOVED*** return aa;
- ***REMOVED*** ***REMOVED***
- ***REMOVED*** ***REMOVED***
- ***REMOVED*** return null;
- ***REMOVED*** ***REMOVED***
-
- @Override
public Set getAlgoFromIs() {
***REMOVED*** TODO Auto-generated method stub
@@ -343,8 +391,6 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED***
-
- @Override
public List updateSVN(String file, List ldep) throws SVNException, IOException {
***REMOVED*** TODO Auto-generated method stub
return null;
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java
new file mode 100644
index 0000000..0e1f345
--- /dev/null
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java
@@ -0,0 +1,52 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
+
+***REMOVED***
+
+/**
+ * Created by ggiammat on 5/9/17.
+ */
+public class AlgorithmBuilder {
+
+
+ public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException {
+ return create(algorithmPackageURL, null, null, null, null, null, null, null);
+ ***REMOVED***
+
+ public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description,
+ String category, String algorithmType, String skipJava) throws IOException, InterruptedException {
+ Algorithm algo = new AlgorithmPackageParser().parsePackage(algorithmPackageURL);
+
+
+ if(category != null){
+ algo.setCategory(category);
+ ***REMOVED***
+
+ ***REMOVED***FIXME: do the same done for category
+ if (algo.getAlgorithmType() == null) {
+ algo.setAlgorithmType(algorithmType);
+ ***REMOVED*** else
+ algo.setAlgorithmType(algo.getCategory());
+
+ if (algo.getSkipJava() == null) {
+ algo.setSkipJava(skipJava);
+ ***REMOVED*** else
+ algo.setSkipJava(algo.getSkipJava());
+
+ if (algo.getName() == null) {
+ algo.setName(name);
+ ***REMOVED*** else
+ algo.setName(algo.getName());
+
+ if (algo.getDescription() == null) {
+ algo.setDescription(description);
+ ;
+ ***REMOVED*** else
+ algo.setDescription(algo.getDescription());
+
+ return algo;
+ ***REMOVED***
+
+***REMOVED***