diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java index fc3cd26..91ccf54 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java @@ -5,6 +5,8 @@ import java.io.BufferedReader; import java.io.InputStreamReader; ***REMOVED*** import java.util.LinkedList; +***REMOVED*** + ***REMOVED*** ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; @@ -74,7 +76,11 @@ public class HAProxy ***REMOVED*** ***REMOVED*** - public List listDataMinersByCluster() throws IOException ***REMOVED*** + public List listDataMinersByCluster(String targetVREToken,String targetVRE) throws IOException ***REMOVED*** + + SecurityTokenProvider.instance.set(targetVREToken); + ScopeProvider.instance.set(targetVRE); + ***REMOVED*** next op to use when Cluster info available in the IS ***REMOVED*** Cluster cluster = this.getClusterByHProxy(); Cluster cluster = this.MapCluster(); @@ -94,9 +100,9 @@ public class HAProxy ***REMOVED*** ***REMOVED******REMOVED*** else ***REMOVED*** ***REMOVED*** prod - ***REMOVED***URL stockURL = new - ***REMOVED***URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); - URL stockURL = new URL("http:***REMOVED***"+ ISClient.getHProxy() +":8880/;csv"); + URL stockURL = new + URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); + ***REMOVED***URL stockURL = new URL("http:***REMOVED***"+ ISClient.getHProxy() +":8880/;csv"); ***REMOVED***URL stockURL = new URL("http:***REMOVED***data.d4science.org/c29KTUluTkZnRlB0WXE5NVNaZnRoR0dtYThUSmNTVlhHbWJQNStIS0N6Yz0"); ***REMOVED***System.out.println(stockURL); ***REMOVED*** dev @@ -124,11 +130,14 @@ public class HAProxy ***REMOVED*** public static void main(String[] args) throws IOException, SVNException ***REMOVED*** HAProxy a = new HAProxy(); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); + ***REMOVED***ScopeProvider.instance.set("/gcube/devNext/NextNext"); + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + ***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***"); + ***REMOVED***ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); ***REMOVED*** System.out.println(a.getHProxy()); ***REMOVED*** System.out.println(a.MapCluster()); - System.out.println(a.listDataMinersByCluster()); + System.out.println(a.listDataMinersByCluster("***REMOVED***","/gcube/devNext/NextNext")); ***REMOVED*** System.out.println(a.listDataMinersByCluster()); ***REMOVED*** List list = new LinkedList(); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java index bccad38..60b9cb8 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java @@ -12,11 +12,11 @@ public class Algorithm ***REMOVED*** private String name; private String description; - - private String category; private String clazz; + private String category; + private String algorithmType; private String skipJava; @@ -31,6 +31,10 @@ public class Algorithm ***REMOVED*** this.actions = new Vector<>(); this.dependencies = new Vector<>(); Dependency p = new Dependency(); + + ***REMOVED***init with default values + this.skipJava = "N"; + this.algorithmType = "transducerers"; ***REMOVED*** public void addDependency(Dependency dep) ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java index 1a0aaa3..73b2090 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java @@ -6,10 +6,15 @@ public class Host ***REMOVED*** private Domain domain; - public Host() ***REMOVED*** + public Host(String hostname) ***REMOVED*** + this.setName(hostname); ***REMOVED*** - public String getFullyQualifiedName() ***REMOVED*** + public Host() ***REMOVED*** + ***REMOVED*** TODO Auto-generated constructor stub +***REMOVED*** + +public String getFullyQualifiedName() ***REMOVED*** if(this.domain!=null && this.domain.getName()!=null) return this.getName()+"."+this.getDomain().getName(); else diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index d7f71f6..25c3812 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -44,8 +44,8 @@ public interface PoolManager ***REMOVED*** String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException; - String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException; - String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException; + String stageAlgorithm(String algorithmPackageURL, String category) throws IOException, InterruptedException; + String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE, String category) throws IOException, InterruptedException; Algorithm extractAlgorithm(String url) throws IOException; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java new file mode 100644 index 0000000..dd4565b --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java @@ -0,0 +1,100 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.service; + +import java.io.File; +import java.io.FileOutputStream; +***REMOVED*** +import java.io.PrintStream; +import java.util.LinkedList; +***REMOVED*** +import java.util.UUID; + +import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; +import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +***REMOVED*** + +public abstract class DMPMJob ***REMOVED*** + + + protected SVNUpdater svnUpdater; + protected File jobLogs; + + protected String id; + + public DMPMJob(SVNUpdater svnUpdater)***REMOVED*** + this.svnUpdater = svnUpdater; + this.id = UUID.randomUUID().toString(); + + ***REMOVED***TODO: dmpm work directory should be loaded from configuration file + this.jobLogs = new File(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"+File.separator+"jobs"); + + this.jobLogs.mkdirs(); +***REMOVED*** + + + public String start()***REMOVED*** + new Thread(new Runnable() ***REMOVED*** + ***REMOVED*** + public void run() ***REMOVED*** + try ***REMOVED*** + execute(); + ***REMOVED*** catch (Exception e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** + ***REMOVED***).start(); + + return this.id; +***REMOVED*** + + protected AnsibleWorker createWorker(Algorithm algo, + Cluster dataminerCluster, + boolean includeAlgorithmDependencies, + String user)***REMOVED*** + AnsibleBridge ansibleBridge = new AnsibleBridge(); + try ***REMOVED*** + return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); + ***REMOVED*** catch (IOException e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + +***REMOVED*** + + protected abstract void execute(); + + protected int executeAnsibleWorker(AnsibleWorker worker) throws IOException, InterruptedException, SVNException***REMOVED*** + File path = new File(worker.getWorkdir() + File.separator + "jobs"); + path.mkdirs(); + File n = new File(this.jobLogs + File.separator + this.id); + FileOutputStream fos = new FileOutputStream(n); + PrintStream ps = new PrintStream(fos); + return worker.execute(ps); +***REMOVED*** + + + protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED*** + for (Dependency d : algo.getDependencies()) ***REMOVED*** + + if (d.getType().equals("os")) ***REMOVED*** + List ls = new LinkedList(); + ls.add(d.getName()); + this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("cran")) ***REMOVED*** + List ls = new LinkedList(); + ls.add(d.getName()); + this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("github")) ***REMOVED*** + List ls = new LinkedList(); + ls.add(d.getName()); + this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls); + ***REMOVED*** + ***REMOVED*** +***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index bd120a1..a96f9be 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -1,40 +1,20 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; +import java.io.File; +import java.io.FileNotFoundException; ***REMOVED*** -import org.gcube.common.resources.gcore.GenericResource; -import org.gcube.common.resources.gcore.Resources; +import java.util.Scanner; + ***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.Props; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; -import org.gcube.informationsystem.publisher.RegistryPublisherFactory; -import org.gcube.informationsystem.publisher.ScopedPublisher; -import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; ***REMOVED*** -***REMOVED*** -***REMOVED*** - -import java.io.*; -***REMOVED*** -***REMOVED*** -import java.net.URLConnection; -***REMOVED*** -import java.util.*; - -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; public class DataminerPoolManager ***REMOVED*** - private SVNUpdater svnUpdater; public DataminerPoolManager()***REMOVED*** @@ -50,260 +30,30 @@ public class DataminerPoolManager ***REMOVED*** ***REMOVED***Algorithm algo) throws IOException, InterruptedException ***REMOVED*** - - Cluster cluster = getStagingDataminerCluster(); - - ***REMOVED***Assumes the service is running in RPrototypingLab - String token = SecurityTokenProvider.instance.get(); - - return addAlgorithmToCluster(algo, cluster, true, "root", true, token); + Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); + Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); + + DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster); + String id = job.start(); + return id; ***REMOVED*** -***REMOVED***Algorithm algo, String targetVREToken) throws IOException, InterruptedException ***REMOVED*** - Cluster cluster = new Cluster(); - SecurityTokenProvider.instance.set(targetVREToken); - for (Host h : new HAProxy().listDataMinersByCluster()) ***REMOVED*** - cluster.addHost(h); - ***REMOVED*** - return addAlgorithmToCluster(algo, cluster, false, "gcube", false, targetVREToken); - -***REMOVED*** - - ***REMOVED***1. to complete - private Cluster getStagingDataminerCluster()***REMOVED*** - Cluster cluster = new Cluster(); - Host h = new Host(); - Props p = new Props(); - ***REMOVED***TODO: read this from configuration or IS? - h.setName(p.getStagingHost()); - cluster.addHost(h); - return cluster; -***REMOVED*** - - - private void createISResource(Algorithm algo, String vreToken) ***REMOVED*** - ISClient client = new ISClient(); - for (Algorithm a : client.getAlgoFromIs()) ***REMOVED*** - if (a.getName().equals(algo.getName())) ***REMOVED*** - continue; - ***REMOVED*** - if (!a.getName().equals(algo.getName()))***REMOVED*** - new ISClient().addAlgToIs(algo, vreToken); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** TODO: create the resource only if not already present +***REMOVED***Algorithm algo, String targetVREToken, String targetVRE) throws IOException, InterruptedException ***REMOVED*** + + Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); + + DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster); + String id = job.start(); + return id; ***REMOVED*** - private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED*** - for (Dependency d : algo.getDependencies()) ***REMOVED*** - - if (d.getType().equals("os")) ***REMOVED*** - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("cran")) ***REMOVED*** - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("github")) ***REMOVED*** - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls); - ***REMOVED*** - ***REMOVED*** + public String getLogById(String id) throws FileNotFoundException***REMOVED*** + + ***REMOVED***TODO: load dir from configuration file + File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/jobs/" + + id); + + return new Scanner(path).useDelimiter("\\Z").next(); ***REMOVED*** - - -***REMOVED**** - * - * @param algo - * @param dataminerCluster - * @return uuid of the execution -***REMOVED*** - public String addAlgorithmToCluster( - final Algorithm algo, - Cluster dataminerCluster, - boolean includeAlgorithmDependencies, - String user, - final boolean stagingVRE, - final String targetVREToken) throws IOException ***REMOVED*** - - AnsibleBridge ansibleBridge = new AnsibleBridge(); - final AnsibleWorker worker = ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); - - new Thread(new Runnable() ***REMOVED*** - ***REMOVED*** - public void run() ***REMOVED*** - try ***REMOVED*** - File path = new File(worker.getWorkdir() + File.separator + "logs"); - path.mkdirs(); - File n = new File(path + File.separator + worker.getWorkerId()); - FileOutputStream fos = new FileOutputStream(n); - PrintStream ps = new PrintStream(fos); - int retValue = worker.execute(ps); - System.out.println("Log stored to to " + n.getAbsolutePath()); - - if(retValue == 0) ***REMOVED*** - updateSVNDependencies(algo, stagingVRE); - ***REMOVED***createISResource(algo, targetVREToken); - ***REMOVED*** - ***REMOVED*** destroy the worker - worker.destroy(); - ***REMOVED*** catch (Exception e) ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); - - ***REMOVED*** this line will execute immediately, not waiting for task to - ***REMOVED*** complete - return worker.getWorkerId(); -***REMOVED*** - - - - public String getScriptFromURL(URL url) throws IOException ***REMOVED*** - if (url == null) ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - URLConnection yc = url.openConnection(); - BufferedReader input = new BufferedReader(new InputStreamReader(yc.getInputStream())); - String line; - StringBuffer buffer = new StringBuffer(); - while ((line = input.readLine()) != null) ***REMOVED*** - buffer.append(line + "\n"); - ***REMOVED*** - String bufferScript = buffer.substring(0, buffer.length()); - input.close(); - return bufferScript; -***REMOVED*** - -***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException ***REMOVED*** -***REMOVED*** ***REMOVED*** create a fake algorithm set -***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet(); -***REMOVED*** algoSet.setName("fake"); -***REMOVED*** algoSet.addAlgorithm(algorithm); -***REMOVED*** final String uuid = UUID.randomUUID().toString(); -***REMOVED*** -***REMOVED*** new Thread(new Runnable() ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** public void run() ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** try ***REMOVED*** -***REMOVED*** try ***REMOVED*** -***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test); -***REMOVED*** ***REMOVED*** catch (SVNException e) ***REMOVED*** -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED*** -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** catch (InterruptedException e) ***REMOVED*** -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED***).start(); -***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for task to -***REMOVED*** ***REMOVED*** complete -***REMOVED*** System.out.println(uuid); -***REMOVED*** return uuid; -***REMOVED******REMOVED*** - -***REMOVED*** public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException ***REMOVED*** -***REMOVED*** ***REMOVED*** create a fake algorithm set -***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet(); -***REMOVED*** algoSet.setName("fake"); -***REMOVED*** algoSet.addAlgorithm(algorithm); -***REMOVED*** final String uuid = UUID.randomUUID().toString(); -***REMOVED*** -***REMOVED*** new Thread(new Runnable() ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** public void run() ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** try ***REMOVED*** -***REMOVED*** if(test)***REMOVED*** -***REMOVED*** addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED*** -***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED*** -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** catch (InterruptedException e) ***REMOVED*** -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** catch (SVNException e) ***REMOVED*** -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED***).start(); -***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for your task to -***REMOVED*** ***REMOVED*** complete -***REMOVED*** System.out.println(uuid); -***REMOVED*** return uuid; -***REMOVED******REMOVED*** - - - public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException ***REMOVED*** - - File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/" - + a + File.separator + "logs"); - path.mkdirs(); - File n = new File(path + File.separator + a); - ***REMOVED*** String addr = InetAddress.getLocalHost().getHostAddress(); - - return new File(n.getPath()).toURI().toURL(); -***REMOVED*** - -***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException ***REMOVED*** -***REMOVED*** -***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre) -***REMOVED*** Cluster cluster = new Cluster(); -***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) ***REMOVED*** -***REMOVED*** ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) ***REMOVED*** -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** ***REMOVED*** apply the changes -***REMOVED*** AnsibleBridge a = new AnsibleBridge(); -***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId(); -***REMOVED*** -***REMOVED******REMOVED*** - -***REMOVED*** public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) -***REMOVED*** throws IOException, InterruptedException, SVNException ***REMOVED*** -***REMOVED*** -***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre) -***REMOVED*** Cluster cluster = new Cluster(); -***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) ***REMOVED*** -***REMOVED*** if (h.getName().equals(hostname)) ***REMOVED*** -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** if(ISClient.getHProxy().equals(hostname))***REMOVED*** -***REMOVED*** ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname)); -***REMOVED*** ***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** apply the changes -***REMOVED*** AnsibleBridge a = new AnsibleBridge(); -***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); -***REMOVED*** -***REMOVED******REMOVED*** - -***REMOVED*** -***REMOVED*** public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) -***REMOVED*** throws IOException, InterruptedException, SVNException ***REMOVED*** -***REMOVED*** Cluster cluster = new Cluster(); -***REMOVED*** Host h = new Host(); -***REMOVED*** h.setName(hostname); -***REMOVED*** cluster.addHost(h); -***REMOVED*** -***REMOVED*** AnsibleBridge a = new AnsibleBridge(); -***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); -***REMOVED*** -***REMOVED******REMOVED*** - - ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java new file mode 100644 index 0000000..ca3c1d5 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -0,0 +1,33 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.service; + +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; + +public class ProductionPublishingJob extends DMPMJob ***REMOVED*** + + private Algorithm algorithm; + private Cluster prodCluster; + + public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster) ***REMOVED*** + super(svnUpdater); + this.algorithm = algorithm; + this.prodCluster = prodCluster; +***REMOVED*** + +***REMOVED*** + protected void execute() ***REMOVED*** + try ***REMOVED*** + this.updateSVNDependencies(this.algorithm, false); + + int ret = this.executeAnsibleWorker( + createWorker(this.algorithm, this.prodCluster, false, "gcube")); + + ***REMOVED*** catch (Exception e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java new file mode 100644 index 0000000..b958a0e --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -0,0 +1,46 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.service; + +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; + +public class StagingJob extends DMPMJob ***REMOVED*** + + + private Algorithm algorithm; + private Cluster stagingCluster; + private Cluster rProtoCluster; + + public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster) ***REMOVED*** + super(svnUpdater); + this.algorithm = algorithm; + this.stagingCluster = stagingCluster; + this.rProtoCluster = rProtoCluster; +***REMOVED*** + +***REMOVED*** + protected void execute() ***REMOVED*** + try ***REMOVED*** + + int ret = this.executeAnsibleWorker( + createWorker(this.algorithm, this.stagingCluster, true, "root")); + + if(ret == 0)***REMOVED*** + + + this.updateSVNDependencies(this.algorithm, true); + + + int ret2 = this.executeAnsibleWorker( + createWorker(this.algorithm, this.rProtoCluster, false, "gcube")); + + ***REMOVED*** + + + + ***REMOVED*** catch (Exception e) ***REMOVED*** + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java index e47675e..00f95c1 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java @@ -11,13 +11,15 @@ import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackagePars public class AlgorithmBuilder ***REMOVED*** - public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException ***REMOVED*** - return create(algorithmPackageURL, null, null, null, null, null, null, null); + public static Algorithm create(String algorithmPackageURL, String category) throws IOException, InterruptedException ***REMOVED*** + return create(algorithmPackageURL, null, null, null, null, category, null, null); ***REMOVED*** public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description, String category, String algorithmType, String skipJava) throws IOException, InterruptedException ***REMOVED*** - Algorithm algo = new AlgorithmPackageParser().parsePackage(algorithmPackageURL); + + + Algorithm algo = new AlgorithmPackageParser().parsePackage(algorithmPackageURL); if(category != null)***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java new file mode 100644 index 0000000..f730218 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java @@ -0,0 +1,44 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +***REMOVED*** + +***REMOVED*** +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; + +public class ClusterBuilder ***REMOVED*** + + + ***REMOVED***1. to complete + public static Cluster getStagingDataminerCluster()***REMOVED*** + Cluster cluster = new Cluster(); + Props p = new Props(); + Host h = new Host(p.getStagingHost()); + ***REMOVED***TODO: read this from configuration or IS? + h.setName(p.getStagingHost()); + cluster.addHost(h); + return cluster; +***REMOVED*** + + public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException***REMOVED*** + Cluster cluster = new Cluster(); + for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) ***REMOVED*** + cluster.addHost(h); + ***REMOVED*** + return cluster; +***REMOVED*** + + public static Cluster getRProtoCluster() throws IOException***REMOVED*** + ***REMOVED***Assumes the service is running in RPrototypingLab + + String token = SecurityTokenProvider.instance.get(); + String targetVRE = ScopeProvider.instance.get(); + + return getVRECluster(token, targetVRE); +***REMOVED*** + + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index 2dbd688..c40d615 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -25,9 +25,8 @@ public class SVNUpdater ***REMOVED*** private SVNRepository svnRepository; public SVNUpdater(String rootURL) throws SVNException ***REMOVED*** - final SVNRepository svnRepository = this.getSvnRepository( + this.svnRepository = this.getSvnRepository( rootURL); - ***REMOVED*** @@ -35,7 +34,7 @@ public class SVNUpdater ***REMOVED*** SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); repository.setAuthenticationManager(authManager); - ***REMOVED***System.out.println(repository.getLocation()); + System.out.println(repository.getLocation()); return repository; ***REMOVED*** @@ -43,7 +42,7 @@ public class SVNUpdater ***REMOVED*** - public List updateSVN(String file, List ldep) throws SVNException, IOException ***REMOVED*** + public void updateSVN(String file, List ldep) ***REMOVED*** try ***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); @@ -73,9 +72,13 @@ public class SVNUpdater ***REMOVED*** new ByteArrayInputStream(bytes), commitEditor, true); commitEditor.closeFile(file, checksum); commitEditor.closeEdit(); - return aa; + + + ***REMOVED***catch(Exception ex)***REMOVED*** + ex.printStackTrace(); + ***REMOVED*** - ***REMOVED*** finally ***REMOVED*** + finally ***REMOVED*** svnRepository.closeSession(); ***REMOVED*** ***REMOVED*** diff --git a/src/main/resources/service.properties b/src/main/resources/service.properties index aa0b8ea..2011784 100644 --- a/src/main/resources/service.properties +++ b/src/main/resources/service.properties @@ -1,4 +1,4 @@ #YML node file -STAGING_HOST: dataminer1-devnext.d4science.org +STAGING_HOST: dataminer-proto-ghost.d4science.org SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ #HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0 diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java new file mode 100644 index 0000000..db8788a --- /dev/null +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java @@ -0,0 +1,46 @@ +package org.gcube.dataanalysis.dataminerpoolmanager; + +***REMOVED*** + +***REMOVED*** +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; +import org.gcube.dataanalysis.dataminer.poolmanager.service.DMPMJob; +import org.gcube.dataanalysis.dataminer.poolmanager.service.ProductionPublishingJob; +import org.gcube.dataanalysis.dataminer.poolmanager.service.StagingJob; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +***REMOVED*** + +public class JobTest ***REMOVED*** + + public static void main(String[] args) throws SVNException, IOException, InterruptedException***REMOVED*** + + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + + + + SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); + + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); + + + Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); + + + Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); + + + DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster); + + job.start(); + + +***REMOVED*** Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); +***REMOVED*** +***REMOVED*** DMPMJob job2 = new ProductionPublishingJob(svnUpdater, algo, prodCluster); + +***REMOVED*** + +***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java new file mode 100644 index 0000000..97333ef --- /dev/null +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java @@ -0,0 +1,27 @@ +package org.gcube.dataanalysis.dataminerpoolmanager; + +***REMOVED*** +import java.util.LinkedList; +***REMOVED*** + +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +***REMOVED*** + +public class SVNUpdaterTest ***REMOVED*** + + public static void main(String[] args) throws SVNException, IOException ***REMOVED*** + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + + + + SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); + + String test = "testDep"; + List ldep = new LinkedList<>(); + ldep.add(test); + + svnUpdater.updateSVN("test_r_cran_pkgs.txt", ldep); +***REMOVED*** + +***REMOVED***