git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148620 82a268e6-3cf1-43bd-a215-b396298e98cf

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java

@@ -12,11 +12,11 @@ public class Algorithm  {
   private String name;
 
   private String description;
-
-  private String category;
   
   private String clazz;
   
+  private String category;
+
   private String algorithmType;
   
   private String skipJava;
@@ -31,6 +31,10 @@ public class Algorithm  {
     this.actions = new Vector<>();
     this.dependencies = new Vector<>();
     Dependency p = new Dependency();
+    
+    ***REMOVED***init with default values
+    this.skipJava = "N";
+    this.algorithmType = "transducerers";
   ***REMOVED***
   
   public void addDependency(Dependency dep) {

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java

@@ -6,10 +6,15 @@ public class Host {
   
   private Domain domain;
 
-  public Host() {
+  public Host(String hostname) {
+	  this.setName(hostname);
   ***REMOVED***
 
-  public String getFullyQualifiedName() {
+  public Host() {
+	***REMOVED*** TODO Auto-generated constructor stub
+***REMOVED***
+
+public String getFullyQualifiedName() {
     if(this.domain!=null && this.domain.getName()!=null)
       return this.getName()+"."+this.getDomain().getName();
     else

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java

@@ -44,8 +44,8 @@ public interface PoolManager {
   String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException;
 
 
-	String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException;
+	String stageAlgorithm(String algorithmPackageURL, String category) throws IOException, InterruptedException;
-	String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException;
+	String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE, String category) throws IOException, InterruptedException;
 
 
 	Algorithm extractAlgorithm(String url) throws IOException;

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java

@@ -1,15 +1,5 @@
 package org.gcube.dataanalysis.dataminer.poolmanager.rest;
 
-***REMOVED***
-***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
-import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
-import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
-***REMOVED***
-***REMOVED***
-import org.tmatesoft.svn.core.SVNException;
-
-import javax.ws.rs.*;
 ***REMOVED***
 import java.net.MalformedURLException;
 ***REMOVED***
@@ -17,6 +7,18 @@ import java.net.UnknownHostException;
 ***REMOVED***
 import java.util.Set;
 
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
+***REMOVED***
+***REMOVED***
+import org.tmatesoft.svn.core.SVNException;
+
 @Path("/")
 public class RestPoolManager implements PoolManager {
 
@@ -26,12 +28,11 @@ public class RestPoolManager implements PoolManager {
 
 	***REMOVED***TODO: for all methods accept also a callback url to send the log.
 
-
 	@GET
 	@Path("/algorithm/stage")
 	@Produces("text/plain")
-	public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException {
+	public String stageAlgorithm(String algorithmPackageURL, String category) throws IOException, InterruptedException {
-		Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
+		Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL, category);
 		return this.service.stageAlgorithm(algo);
 ***REMOVED***
 
@@ -39,9 +40,9 @@ public class RestPoolManager implements PoolManager {
 	@GET
 	@Path("/algorithm/add")
 	@Produces("text/plain")
-	public String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException {
+	public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE, String category) throws IOException, InterruptedException {
-		Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
+		Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL, category);
-		return this.service.publishAlgorithm(algo, targetVREToken);
+		return this.service.publishAlgorithm(algo, targetVREToken, targetVRE);
 ***REMOVED***
 
 	/*
@@ -55,7 +56,7 @@ public class RestPoolManager implements PoolManager {
 	public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException {
 		***REMOVED*** TODO Auto-generated method stub
 		LOGGER.debug("Returning Log =" + logUrl);
-		return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl));
+		return service.getLogById(logUrl);
 ***REMOVED***
 
 	@Override
@@ -65,17 +66,17 @@ public class RestPoolManager implements PoolManager {
 ***REMOVED***
 
 	public static void main(String[] args) throws IOException, InterruptedException, SVNException {
-		***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
+***REMOVED***		***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
-		***REMOVED*** ProxySelector.setDefault(new
+***REMOVED***		***REMOVED*** ProxySelector.setDefault(new
-		***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
+***REMOVED***		***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
-
+***REMOVED***
-		ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
+***REMOVED***		ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
-		***REMOVED***SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
+***REMOVED***		***REMOVED***SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
-		RestPoolManager a = new RestPoolManager();
+***REMOVED***		RestPoolManager a = new RestPoolManager();
-		***REMOVED***a.stageAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0");
+***REMOVED***		a.stageAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", category);
-		a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
+***REMOVED***		***REMOVED***a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext");
-		***REMOVED*** PoolManager aa = new DataminerPoolManager();
+***REMOVED***		***REMOVED*** PoolManager aa = new DataminerPoolManager();
-
+***REMOVED***
 
 ***REMOVED***
 

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java

@@ -0,0 +1,100 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.service;
+
+import java.io.File;
+import java.io.FileOutputStream;
+***REMOVED***
+import java.io.PrintStream;
+***REMOVED***
+***REMOVED***
+import java.util.UUID;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
+import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.tmatesoft.svn.core.SVNException;
+
+public abstract class DMPMJob {
+	
+	
+	protected SVNUpdater svnUpdater;
+	protected File jobLogs;
+	
+	protected String id;
+	
+	public DMPMJob(SVNUpdater svnUpdater){
+		this.svnUpdater = svnUpdater;
+		this.id = UUID.randomUUID().toString();
+		
+		***REMOVED***TODO: dmpm work directory should be loaded from configuration file
+		this.jobLogs = new File(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"+File.separator+"jobs");
+	
+		this.jobLogs.mkdirs();
+***REMOVED***
+
+	
+	public String start(){
+		new Thread(new Runnable() {
+			@Override
+			public void run() {
+				try {
+					execute();
+			***REMOVED*** catch (Exception e) {
+					e.printStackTrace();
+			***REMOVED***
+		***REMOVED***
+	***REMOVED***).start();
+		
+		return this.id;
+***REMOVED***
+	
+	protected AnsibleWorker createWorker(Algorithm algo,
+			Cluster dataminerCluster,
+			boolean includeAlgorithmDependencies,
+			String user){
+		AnsibleBridge ansibleBridge = new AnsibleBridge();
+		try {
+			return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user);
+	***REMOVED*** catch (IOException e) {
+			e.printStackTrace();
+	***REMOVED***
+		return null;
+		
+***REMOVED***
+	
+	protected abstract void execute();
+	
+	protected int executeAnsibleWorker(AnsibleWorker worker) throws IOException, InterruptedException, SVNException{
+		File path = new File(worker.getWorkdir() + File.separator + "jobs");
+		path.mkdirs();
+		File n = new File(this.jobLogs + File.separator + this.id);
+		FileOutputStream fos = new FileOutputStream(n);
+		PrintStream ps = new PrintStream(fos);
+		return worker.execute(ps);
+***REMOVED***
+	
+	
+	protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
+		for (Dependency d : algo.getDependencies()) {
+
+			if (d.getType().equals("os")) {
+				List<String> ls = new LinkedList<String>();
+				ls.add(d.getName());
+				this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
+		***REMOVED***
+			if (d.getType().equals("cran")) {
+				List<String> ls = new LinkedList<String>();
+				ls.add(d.getName());
+				this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
+		***REMOVED***
+			if (d.getType().equals("github")) {
+				List<String> ls = new LinkedList<String>();
+				ls.add(d.getName());
+				this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
+		***REMOVED***
+	***REMOVED***
+***REMOVED***
+
+***REMOVED***

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java

@@ -1,40 +1,20 @@
 package org.gcube.dataanalysis.dataminer.poolmanager.service;
 
+import java.io.File;
+import java.io.FileNotFoundException;
 ***REMOVED***
-import org.gcube.common.resources.gcore.GenericResource;
+import java.util.Scanner;
-import org.gcube.common.resources.gcore.Resources;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
 ***REMOVED***
-import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
-import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
-import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
-import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
-import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
 import org.gcube.dataanalysis.dataminer.poolmanager.util.Props;
 import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
-import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
-import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
-import org.gcube.informationsystem.publisher.ScopedPublisher;
-import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException;
-import org.gcube.resources.discovery.client.api.DiscoveryClient;
-import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
-***REMOVED***
-***REMOVED***
 import org.tmatesoft.svn.core.SVNException;
 
-import java.io.*;
-import java.net.MalformedURLException;
-***REMOVED***
-import java.net.URLConnection;
-import java.net.UnknownHostException;
-import java.util.*;
-
-import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
-import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
-
 public class DataminerPoolManager {
 
 
-
 	private SVNUpdater svnUpdater;
 
 	public DataminerPoolManager(){
@@ -50,260 +30,30 @@ public class DataminerPoolManager {
 
 	public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
 
-
+		Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
-		Cluster cluster = getStagingDataminerCluster();
+		Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
-
+		
-		***REMOVED***Assumes the service is running in RPrototypingLab
+		DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster);
-		String token = SecurityTokenProvider.instance.get();
+		String id = job.start();
-
+		return id;
-		return addAlgorithmToCluster(algo, cluster, true, "root", true, token);
 ***REMOVED***
 
-	public String publishAlgorithm(Algorithm algo, String targetVREToken) throws IOException, InterruptedException {
+	public String publishAlgorithm(Algorithm algo, String targetVREToken, String targetVRE) throws IOException, InterruptedException {
-		Cluster cluster = new Cluster();
+		
-***REMOVED***
+		Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
-		for (Host h : new HAProxy().listDataMinersByCluster()) {
+		
-			cluster.addHost(h);
+		DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster);
-	***REMOVED***
+		String id = job.start();
-		return addAlgorithmToCluster(algo, cluster, false, "gcube", false, targetVREToken);
+		return id;
-
-***REMOVED***
-
-	***REMOVED***1. to complete
-	private Cluster getStagingDataminerCluster(){
-		Cluster cluster = new Cluster();
-		Host h = new Host();
-		Props p = new Props();
-		***REMOVED***TODO: read this from configuration or IS?
-		h.setName(p.getStagingHost());
-		cluster.addHost(h);
-		return cluster;
-***REMOVED***
-	
-	
-	private void createISResource(Algorithm algo, String vreToken) {
-		ISClient client = new ISClient();
-		for (Algorithm a : client.getAlgoFromIs()) {
-			if (a.getName().equals(algo.getName())) {
-				continue;
-		***REMOVED*** 
-			if (!a.getName().equals(algo.getName())){
-				new ISClient().addAlgToIs(algo, vreToken);
-		***REMOVED***
-	***REMOVED***
-		***REMOVED*** TODO: create the resource only if not already present
 ***REMOVED***
 
 
-	private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
+	public String getLogById(String id) throws FileNotFoundException{
-		for (Dependency d : algo.getDependencies()) {
+		
-
+		***REMOVED***TODO: load dir from configuration file
-			if (d.getType().equals("os")) {
+		File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/jobs/"
-				List<String> ls = new LinkedList<String>();
+				+ id);
-				ls.add(d.getName());
+		
-				this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
+		return new Scanner(path).useDelimiter("\\Z").next();
-		***REMOVED***
-			if (d.getType().equals("cran")) {
-				List<String> ls = new LinkedList<String>();
-				ls.add(d.getName());
-				this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
-		***REMOVED***
-			if (d.getType().equals("github")) {
-				List<String> ls = new LinkedList<String>();
-				ls.add(d.getName());
-				this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
-		***REMOVED***
-	***REMOVED***
 ***REMOVED***
-
-
-	/**
-	 *
-	 * @param algo
-	 * @param dataminerCluster
-	 * @return uuid of the execution
-	 */
-	public String addAlgorithmToCluster(
-			final Algorithm algo,
-			Cluster dataminerCluster,
-			boolean includeAlgorithmDependencies,
-			String user,
-			final boolean stagingVRE,
-			final String targetVREToken) throws IOException {
-
-		AnsibleBridge ansibleBridge = new AnsibleBridge();
-		final AnsibleWorker worker = ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user);
-
-		new Thread(new Runnable() {
-			@Override
-			public void run() {
-				try {
-					File path = new File(worker.getWorkdir() + File.separator + "logs");
-					path.mkdirs();
-					File n = new File(path + File.separator + worker.getWorkerId());
-					FileOutputStream fos = new FileOutputStream(n);
-					PrintStream ps = new PrintStream(fos);
-					int retValue = worker.execute(ps);
-					System.out.println("Log stored to to " + n.getAbsolutePath());
-
-					if(retValue == 0) {
-							updateSVNDependencies(algo, stagingVRE);
-							***REMOVED***createISResource(algo, targetVREToken);
-				***REMOVED***
-					***REMOVED*** destroy the worker
-					worker.destroy();
-			***REMOVED*** catch (Exception e) {
-					e.printStackTrace();
-			***REMOVED***
-		***REMOVED***
-	***REMOVED***).start();
-
-		***REMOVED*** this line will execute immediately, not waiting for task to
-		***REMOVED*** complete
-		return worker.getWorkerId();
-***REMOVED***
-
-
-	
-	public String getScriptFromURL(URL url) throws IOException {
-		if (url == null) {
-			return null;
-	***REMOVED***
-		URLConnection yc = url.openConnection();
-		BufferedReader input = new BufferedReader(new InputStreamReader(yc.getInputStream()));
-		String line;
-		StringBuffer buffer = new StringBuffer();
-		while ((line = input.readLine()) != null) {
-			buffer.append(line + "\n");
-	***REMOVED***
-		String bufferScript = buffer.substring(0, buffer.length());
-		input.close();
-		return bufferScript;
-***REMOVED***
-
-***REMOVED***	public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
-***REMOVED***		***REMOVED*** create a fake algorithm set
-***REMOVED***		final AlgorithmSet algoSet = new AlgorithmSet();
-***REMOVED***		algoSet.setName("fake");
-***REMOVED***		algoSet.addAlgorithm(algorithm);
-***REMOVED***		final String uuid = UUID.randomUUID().toString();
-***REMOVED***
-***REMOVED***		new Thread(new Runnable() {
-***REMOVED***			@Override
-***REMOVED***			public void run() {
-***REMOVED***				***REMOVED*** TODO Auto-generated method stub
-***REMOVED***				try {
-***REMOVED***					try {
-***REMOVED***						addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test);
-***REMOVED***				***REMOVED*** catch (SVNException e) {
-***REMOVED***						***REMOVED*** TODO Auto-generated catch block
-***REMOVED***						e.printStackTrace();
-***REMOVED***				***REMOVED***
-***REMOVED***			***REMOVED*** catch (IOException e) {
-***REMOVED***					***REMOVED*** TODO Auto-generated catch block
-***REMOVED***					e.printStackTrace();
-***REMOVED***			***REMOVED*** catch (InterruptedException e) {
-***REMOVED***					***REMOVED*** TODO Auto-generated catch block
-***REMOVED***					e.printStackTrace();
-***REMOVED***			***REMOVED***
-***REMOVED***		***REMOVED***
-***REMOVED***	***REMOVED***).start();
-***REMOVED***		***REMOVED*** this line will execute immediately, not waiting for task to
-***REMOVED***		***REMOVED*** complete
-***REMOVED***		System.out.println(uuid);
-***REMOVED***		return uuid;
-***REMOVED******REMOVED***
-
-***REMOVED***	public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException {
-***REMOVED***		***REMOVED*** create a fake algorithm set
-***REMOVED***		final AlgorithmSet algoSet = new AlgorithmSet();
-***REMOVED***		algoSet.setName("fake");
-***REMOVED***		algoSet.addAlgorithm(algorithm);
-***REMOVED***		final String uuid = UUID.randomUUID().toString();
-***REMOVED***
-***REMOVED***		new Thread(new Runnable() {
-***REMOVED***			@Override
-***REMOVED***			public void run() {
-***REMOVED***				***REMOVED*** TODO Auto-generated method stub
-***REMOVED***				try {
-***REMOVED***					if(test){
-***REMOVED***					addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED***
-***REMOVED***			***REMOVED*** catch (IOException e) {
-***REMOVED***					***REMOVED*** TODO Auto-generated catch block
-***REMOVED***					e.printStackTrace();
-***REMOVED***			***REMOVED*** catch (InterruptedException e) {
-***REMOVED***					***REMOVED*** TODO Auto-generated catch block
-***REMOVED***					e.printStackTrace();
-***REMOVED***			***REMOVED*** catch (SVNException e) {
-***REMOVED***					***REMOVED*** TODO Auto-generated catch block
-***REMOVED***					e.printStackTrace();
-***REMOVED***			***REMOVED***
-***REMOVED***		***REMOVED***
-***REMOVED***	***REMOVED***).start();
-***REMOVED***		***REMOVED*** this line will execute immediately, not waiting for your task to
-***REMOVED***		***REMOVED*** complete
-***REMOVED***		System.out.println(uuid);
-***REMOVED***		return uuid;
-***REMOVED******REMOVED***
-
-
-	public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
-
-		File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/"
-				+ a + File.separator + "logs");
-		path.mkdirs();
-		File n = new File(path + File.separator + a);
-		***REMOVED*** String addr = InetAddress.getLocalHost().getHostAddress();
-
-		return new File(n.getPath()).toURI().toURL();
-***REMOVED***
-
-***REMOVED***	public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException {
-***REMOVED***
-***REMOVED***		***REMOVED*** create the cluster (dataminers in the vre)
-***REMOVED***		Cluster cluster = new Cluster();
-***REMOVED***		 for (Host h : new HAProxy().listDataMinersByCluster()) {
-***REMOVED***		***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) {
-***REMOVED***			cluster.addHost(h);
-***REMOVED***	***REMOVED***
-***REMOVED***
-***REMOVED***		***REMOVED*** apply the changes
-***REMOVED***		AnsibleBridge a = new AnsibleBridge();
-***REMOVED***		return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId();
-***REMOVED***
-***REMOVED******REMOVED***
-
-***REMOVED***	public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
-***REMOVED***			throws IOException, InterruptedException, SVNException {
-***REMOVED***
-***REMOVED***		***REMOVED*** create the cluster (dataminers in the vre)
-***REMOVED***		Cluster cluster = new Cluster();
-***REMOVED***		for (Host h : new HAProxy().listDataMinersByCluster()) {
-***REMOVED***			if (h.getName().equals(hostname)) {
-***REMOVED***				cluster.addHost(h);
-***REMOVED***		***REMOVED***
-***REMOVED***	***REMOVED***
-***REMOVED***		***REMOVED*** if(ISClient.getHProxy().equals(hostname)){
-***REMOVED***		***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname));
-***REMOVED***		***REMOVED*** ***REMOVED***
-***REMOVED***		***REMOVED*** apply the changes
-***REMOVED***		AnsibleBridge a = new AnsibleBridge();
-***REMOVED***		return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
-***REMOVED***
-***REMOVED******REMOVED***
-	
-***REMOVED***	
-***REMOVED***	public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test)
-***REMOVED***			throws IOException, InterruptedException, SVNException {
-***REMOVED***		Cluster cluster = new Cluster();
-***REMOVED***		Host h = new Host();
-***REMOVED***		h.setName(hostname);
-***REMOVED***		cluster.addHost(h);
-***REMOVED***		
-***REMOVED***		AnsibleBridge a = new AnsibleBridge();
-***REMOVED***		return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId();
-***REMOVED***
-***REMOVED******REMOVED***
-
-
 ***REMOVED***

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java

@@ -0,0 +1,33 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.service;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+
+public class ProductionPublishingJob extends DMPMJob {
+	
+	private Algorithm algorithm;
+	private Cluster prodCluster;
+
+	public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster) {
+		super(svnUpdater);
+		this.algorithm = algorithm;
+		this.prodCluster = prodCluster;
+***REMOVED***
+
+	@Override
+	protected void execute() {
+		try {
+			this.updateSVNDependencies(this.algorithm, false);
+
+			int ret = this.executeAnsibleWorker(
+					createWorker(this.algorithm, this.prodCluster, false, "gcube"));	
+		
+	***REMOVED*** catch (Exception e) {
+			e.printStackTrace();
+	***REMOVED***		
+***REMOVED***
+
+	
+	
+***REMOVED***

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java

@@ -0,0 +1,46 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.service;
+
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+
+public class StagingJob extends DMPMJob {
+	
+	
+	private Algorithm algorithm;
+	private Cluster stagingCluster;
+	private Cluster rProtoCluster;
+
+	public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster) {
+		super(svnUpdater);
+		this.algorithm = algorithm;
+		this.stagingCluster = stagingCluster;
+		this.rProtoCluster = rProtoCluster;
+***REMOVED***
+
+	@Override
+	protected void execute() {
+		try {
+			
+			int ret = this.executeAnsibleWorker(
+					createWorker(this.algorithm, this.stagingCluster, true, "root"));
+		
+			if(ret == 0){
+				
+				
+				this.updateSVNDependencies(this.algorithm, true);
+				
+				
+				int ret2 = this.executeAnsibleWorker(
+						createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
+				
+		***REMOVED***
+		
+		
+		
+	***REMOVED*** catch (Exception e) {
+			e.printStackTrace();
+	***REMOVED***
+***REMOVED***
+
+***REMOVED***

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java

@@ -11,13 +11,15 @@ import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackagePars
 public class AlgorithmBuilder {
 
 
-  public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException {
+  public static Algorithm create(String algorithmPackageURL, String category) throws IOException, InterruptedException {
-    return create(algorithmPackageURL, null, null, null, null, null, null, null);
+    return create(algorithmPackageURL, null, null, null, null, category, null, null);
   ***REMOVED***
 
   public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description,
                                  String category, String algorithmType, String skipJava) throws IOException, InterruptedException {
-    Algorithm algo =  new AlgorithmPackageParser().parsePackage(algorithmPackageURL);
+    
+	  
+	  Algorithm algo =  new AlgorithmPackageParser().parsePackage(algorithmPackageURL);
 
 
     if(category != null){

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java

@@ -0,0 +1,44 @@
+package org.gcube.dataanalysis.dataminer.poolmanager.util;
+
+***REMOVED***
+
+***REMOVED***
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
+***REMOVED***
+***REMOVED***
+
+public class ClusterBuilder {
+
+	
+	***REMOVED***1. to complete
+	public static Cluster getStagingDataminerCluster(){
+		Cluster cluster = new Cluster();
+		Props p = new Props();
+		Host h = new Host(p.getStagingHost());
+		***REMOVED***TODO: read this from configuration or IS?
+		h.setName(p.getStagingHost());
+		cluster.addHost(h);
+		return cluster;
+***REMOVED***
+	
+	public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException{
+		Cluster cluster = new Cluster();
+		for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) {
+			cluster.addHost(h);
+	***REMOVED***
+		return  cluster;
+***REMOVED***
+	
+	public static Cluster getRProtoCluster() throws IOException{
+		***REMOVED***Assumes the service is running in RPrototypingLab
+
+		String token = SecurityTokenProvider.instance.get();
+		String targetVRE = ScopeProvider.instance.get();
+		
+		return getVRECluster(token, targetVRE);
+***REMOVED***
+	
+	
+	
+***REMOVED***

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java

@@ -25,9 +25,8 @@ public class SVNUpdater {
   private SVNRepository svnRepository;
 
   public SVNUpdater(String rootURL) throws SVNException {
-    final SVNRepository svnRepository = this.getSvnRepository(
+    this.svnRepository = this.getSvnRepository(
         rootURL);
-
   ***REMOVED***
 
 
@@ -35,7 +34,7 @@ public class SVNUpdater {
     SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
     ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
     repository.setAuthenticationManager(authManager);
-    ***REMOVED***System.out.println(repository.getLocation());
+    System.out.println(repository.getLocation());
 
     return repository;
   ***REMOVED***
@@ -43,7 +42,7 @@ public class SVNUpdater {
 
 
 
-  public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
+  public void updateSVN(String file, List<String> ldep) {
     try {
       final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
       svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
@@ -73,9 +72,13 @@ public class SVNUpdater {
           new ByteArrayInputStream(bytes), commitEditor, true);
       commitEditor.closeFile(file, checksum);
       commitEditor.closeEdit();
-      return aa;
+   
+      
+    ***REMOVED***catch(Exception ex){
+		  ex.printStackTrace();
+	  ***REMOVED***
 
-    ***REMOVED*** finally {
+    finally {
       svnRepository.closeSession();
     ***REMOVED***
   ***REMOVED***

src/main/resources/service.properties

@@ -1,4 +1,4 @@
 #YML node file
-STAGING_HOST: dataminer1-devnext.d4science.org
+STAGING_HOST:  dataminer-proto-ghost.d4science.org
 SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
 #HAPROXY_CSV:  http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0

src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java

@@ -0,0 +1,46 @@
+package org.gcube.dataanalysis.dataminerpoolmanager;
+
+***REMOVED***
+
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.service.DMPMJob;
+import org.gcube.dataanalysis.dataminer.poolmanager.service.ProductionPublishingJob;
+import org.gcube.dataanalysis.dataminer.poolmanager.service.StagingJob;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.tmatesoft.svn.core.SVNException;
+
+public class JobTest {
+	
+	public static void main(String[] args) throws SVNException, IOException, InterruptedException{
+		
+		ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
+
+		
+	
+		SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
+		
+		Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
+		
+		
+		Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
+		
+		
+		Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();		
+	
+		
+		DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster);
+
+		job.start();
+		
+		
+***REMOVED***		Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);		
+***REMOVED***
+***REMOVED***		DMPMJob job2 = new ProductionPublishingJob(svnUpdater, algo, prodCluster);
+		
+***REMOVED***
+
+***REMOVED***

src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java

@@ -0,0 +1,27 @@
+package org.gcube.dataanalysis.dataminerpoolmanager;
+
+***REMOVED***
+***REMOVED***
+***REMOVED***
+
+***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
+import org.tmatesoft.svn.core.SVNException;
+
+public class SVNUpdaterTest {
+
+	public static void main(String[] args) throws SVNException, IOException {
+		ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
+
+		
+		
+		SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
+
+		String test = "testDep";
+		List<String> ldep = new LinkedList<>();
+		ldep.add(test);
+		
+		svnUpdater.updateSVN("test_r_cran_pkgs.txt", ldep);
+***REMOVED***
+
+***REMOVED***