diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java index 2d91e27..62dc3d2 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java @@ -12,11 +12,11 @@ public class Algorithm { private String name; private String description; - - private String category; private String clazz; + private String category; + private String algorithmType; private String skipJava; @@ -31,6 +31,10 @@ public class Algorithm { this.actions = new Vector<>(); this.dependencies = new Vector<>(); Dependency p = new Dependency(); + + ***REMOVED***init with default values + this.skipJava = "N"; + this.algorithmType = "transducerers"; ***REMOVED*** public void addDependency(Dependency dep) { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java index 60348bd..c13b941 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java @@ -6,10 +6,15 @@ public class Host { private Domain domain; - public Host() { + public Host(String hostname) { + this.setName(hostname); ***REMOVED*** - public String getFullyQualifiedName() { + public Host() { + ***REMOVED*** TODO Auto-generated constructor stub +***REMOVED*** + +public String getFullyQualifiedName() { if(this.domain!=null && this.domain.getName()!=null) return this.getName()+"."+this.getDomain().getName(); else diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index 600a3be..2ab7fbf 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -44,8 +44,8 @@ public interface PoolManager { String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException; - String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException; - String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException; + String stageAlgorithm(String algorithmPackageURL, String category) throws IOException, InterruptedException; + String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE, String category) throws IOException, InterruptedException; Algorithm extractAlgorithm(String url) throws IOException; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index 2d7fae0..4d6c5f5 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -1,15 +1,5 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest; -***REMOVED*** -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; -***REMOVED*** -***REMOVED*** -import org.tmatesoft.svn.core.SVNException; - -import javax.ws.rs.*; ***REMOVED*** import java.net.MalformedURLException; ***REMOVED*** @@ -17,6 +7,18 @@ import java.net.UnknownHostException; ***REMOVED*** import java.util.Set; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; + +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; +import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; +***REMOVED*** +***REMOVED*** +import org.tmatesoft.svn.core.SVNException; + @Path("/") public class RestPoolManager implements PoolManager { @@ -26,12 +28,11 @@ public class RestPoolManager implements PoolManager { ***REMOVED***TODO: for all methods accept also a callback url to send the log. - @GET @Path("/algorithm/stage") @Produces("text/plain") - public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException { - Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); + public String stageAlgorithm(String algorithmPackageURL, String category) throws IOException, InterruptedException { + Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL, category); return this.service.stageAlgorithm(algo); ***REMOVED*** @@ -39,9 +40,9 @@ public class RestPoolManager implements PoolManager { @GET @Path("/algorithm/add") @Produces("text/plain") - public String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException { - Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); - return this.service.publishAlgorithm(algo, targetVREToken); + public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE, String category) throws IOException, InterruptedException { + Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL, category); + return this.service.publishAlgorithm(algo, targetVREToken, targetVRE); ***REMOVED*** /* @@ -55,7 +56,7 @@ public class RestPoolManager implements PoolManager { public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException { ***REMOVED*** TODO Auto-generated method stub LOGGER.debug("Returning Log =" + logUrl); - return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl)); + return service.getLogById(logUrl); ***REMOVED*** @Override @@ -65,17 +66,17 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** public static void main(String[] args) throws IOException, InterruptedException, SVNException { - ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); - ***REMOVED*** ProxySelector.setDefault(new - ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - - ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); - ***REMOVED***SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); - RestPoolManager a = new RestPoolManager(); - ***REMOVED***a.stageAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0"); - a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); - ***REMOVED*** PoolManager aa = new DataminerPoolManager(); - +***REMOVED*** ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); +***REMOVED*** ***REMOVED*** ProxySelector.setDefault(new +***REMOVED*** ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); +***REMOVED*** +***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); +***REMOVED*** ***REMOVED***SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); +***REMOVED*** RestPoolManager a = new RestPoolManager(); +***REMOVED*** a.stageAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", category); +***REMOVED*** ***REMOVED***a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext"); +***REMOVED*** ***REMOVED*** PoolManager aa = new DataminerPoolManager(); +***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java new file mode 100644 index 0000000..5d203e5 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java @@ -0,0 +1,100 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.service; + +import java.io.File; +import java.io.FileOutputStream; +***REMOVED*** +import java.io.PrintStream; +***REMOVED*** +***REMOVED*** +import java.util.UUID; + +import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; +import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.tmatesoft.svn.core.SVNException; + +public abstract class DMPMJob { + + + protected SVNUpdater svnUpdater; + protected File jobLogs; + + protected String id; + + public DMPMJob(SVNUpdater svnUpdater){ + this.svnUpdater = svnUpdater; + this.id = UUID.randomUUID().toString(); + + ***REMOVED***TODO: dmpm work directory should be loaded from configuration file + this.jobLogs = new File(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"+File.separator+"jobs"); + + this.jobLogs.mkdirs(); +***REMOVED*** + + + public String start(){ + new Thread(new Runnable() { + @Override + public void run() { + try { + execute(); + ***REMOVED*** catch (Exception e) { + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** + ***REMOVED***).start(); + + return this.id; +***REMOVED*** + + protected AnsibleWorker createWorker(Algorithm algo, + Cluster dataminerCluster, + boolean includeAlgorithmDependencies, + String user){ + AnsibleBridge ansibleBridge = new AnsibleBridge(); + try { + return ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); + ***REMOVED*** catch (IOException e) { + e.printStackTrace(); + ***REMOVED*** + return null; + +***REMOVED*** + + protected abstract void execute(); + + protected int executeAnsibleWorker(AnsibleWorker worker) throws IOException, InterruptedException, SVNException{ + File path = new File(worker.getWorkdir() + File.separator + "jobs"); + path.mkdirs(); + File n = new File(this.jobLogs + File.separator + this.id); + FileOutputStream fos = new FileOutputStream(n); + PrintStream ps = new PrintStream(fos); + return worker.execute(ps); +***REMOVED*** + + + protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException { + for (Dependency d : algo.getDependencies()) { + + if (d.getType().equals("os")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("cran")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("github")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls); + ***REMOVED*** + ***REMOVED*** +***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 0e66f9d..9f86312 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -1,40 +1,20 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; +import java.io.File; +import java.io.FileNotFoundException; ***REMOVED*** -import org.gcube.common.resources.gcore.GenericResource; -import org.gcube.common.resources.gcore.Resources; +import java.util.Scanner; + +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; ***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.Props; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; -import org.gcube.informationsystem.publisher.RegistryPublisherFactory; -import org.gcube.informationsystem.publisher.ScopedPublisher; -import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; -***REMOVED*** -***REMOVED*** import org.tmatesoft.svn.core.SVNException; -import java.io.*; -import java.net.MalformedURLException; -***REMOVED*** -import java.net.URLConnection; -import java.net.UnknownHostException; -import java.util.*; - -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - public class DataminerPoolManager { - private SVNUpdater svnUpdater; public DataminerPoolManager(){ @@ -50,260 +30,30 @@ public class DataminerPoolManager { public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException { - - Cluster cluster = getStagingDataminerCluster(); - - ***REMOVED***Assumes the service is running in RPrototypingLab - String token = SecurityTokenProvider.instance.get(); - - return addAlgorithmToCluster(algo, cluster, true, "root", true, token); + Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); + Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); + + DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster); + String id = job.start(); + return id; ***REMOVED*** - public String publishAlgorithm(Algorithm algo, String targetVREToken) throws IOException, InterruptedException { - Cluster cluster = new Cluster(); -***REMOVED*** - for (Host h : new HAProxy().listDataMinersByCluster()) { - cluster.addHost(h); - ***REMOVED*** - return addAlgorithmToCluster(algo, cluster, false, "gcube", false, targetVREToken); - -***REMOVED*** - - ***REMOVED***1. to complete - private Cluster getStagingDataminerCluster(){ - Cluster cluster = new Cluster(); - Host h = new Host(); - Props p = new Props(); - ***REMOVED***TODO: read this from configuration or IS? - h.setName(p.getStagingHost()); - cluster.addHost(h); - return cluster; -***REMOVED*** - - - private void createISResource(Algorithm algo, String vreToken) { - ISClient client = new ISClient(); - for (Algorithm a : client.getAlgoFromIs()) { - if (a.getName().equals(algo.getName())) { - continue; - ***REMOVED*** - if (!a.getName().equals(algo.getName())){ - new ISClient().addAlgToIs(algo, vreToken); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** TODO: create the resource only if not already present + public String publishAlgorithm(Algorithm algo, String targetVREToken, String targetVRE) throws IOException, InterruptedException { + + Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); + + DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster); + String id = job.start(); + return id; ***REMOVED*** - private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException { - for (Dependency d : algo.getDependencies()) { - - if (d.getType().equals("os")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("cran")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("github")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls); - ***REMOVED*** - ***REMOVED*** + public String getLogById(String id) throws FileNotFoundException{ + + ***REMOVED***TODO: load dir from configuration file + File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/jobs/" + + id); + + return new Scanner(path).useDelimiter("\\Z").next(); ***REMOVED*** - - - /** - * - * @param algo - * @param dataminerCluster - * @return uuid of the execution - */ - public String addAlgorithmToCluster( - final Algorithm algo, - Cluster dataminerCluster, - boolean includeAlgorithmDependencies, - String user, - final boolean stagingVRE, - final String targetVREToken) throws IOException { - - AnsibleBridge ansibleBridge = new AnsibleBridge(); - final AnsibleWorker worker = ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); - - new Thread(new Runnable() { - @Override - public void run() { - try { - File path = new File(worker.getWorkdir() + File.separator + "logs"); - path.mkdirs(); - File n = new File(path + File.separator + worker.getWorkerId()); - FileOutputStream fos = new FileOutputStream(n); - PrintStream ps = new PrintStream(fos); - int retValue = worker.execute(ps); - System.out.println("Log stored to to " + n.getAbsolutePath()); - - if(retValue == 0) { - updateSVNDependencies(algo, stagingVRE); - ***REMOVED***createISResource(algo, targetVREToken); - ***REMOVED*** - ***REMOVED*** destroy the worker - worker.destroy(); - ***REMOVED*** catch (Exception e) { - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); - - ***REMOVED*** this line will execute immediately, not waiting for task to - ***REMOVED*** complete - return worker.getWorkerId(); -***REMOVED*** - - - - public String getScriptFromURL(URL url) throws IOException { - if (url == null) { - return null; - ***REMOVED*** - URLConnection yc = url.openConnection(); - BufferedReader input = new BufferedReader(new InputStreamReader(yc.getInputStream())); - String line; - StringBuffer buffer = new StringBuffer(); - while ((line = input.readLine()) != null) { - buffer.append(line + "\n"); - ***REMOVED*** - String bufferScript = buffer.substring(0, buffer.length()); - input.close(); - return bufferScript; -***REMOVED*** - -***REMOVED*** public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException { -***REMOVED*** ***REMOVED*** create a fake algorithm set -***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet(); -***REMOVED*** algoSet.setName("fake"); -***REMOVED*** algoSet.addAlgorithm(algorithm); -***REMOVED*** final String uuid = UUID.randomUUID().toString(); -***REMOVED*** -***REMOVED*** new Thread(new Runnable() { -***REMOVED*** @Override -***REMOVED*** public void run() { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** try { -***REMOVED*** try { -***REMOVED*** addAlgorithmsToVRE(algoSet, vre, uuid, /*updateSVN*/test); -***REMOVED*** ***REMOVED*** catch (SVNException e) { -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** catch (IOException e) { -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** catch (InterruptedException e) { -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED***).start(); -***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for task to -***REMOVED*** ***REMOVED*** complete -***REMOVED*** System.out.println(uuid); -***REMOVED*** return uuid; -***REMOVED******REMOVED*** - -***REMOVED*** public String addAlgorithmToHost(Algorithm algorithm, final String hostname, /*final boolean updateSVN*/ final boolean test) throws IOException { -***REMOVED*** ***REMOVED*** create a fake algorithm set -***REMOVED*** final AlgorithmSet algoSet = new AlgorithmSet(); -***REMOVED*** algoSet.setName("fake"); -***REMOVED*** algoSet.addAlgorithm(algorithm); -***REMOVED*** final String uuid = UUID.randomUUID().toString(); -***REMOVED*** -***REMOVED*** new Thread(new Runnable() { -***REMOVED*** @Override -***REMOVED*** public void run() { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** try { -***REMOVED*** if(test){ -***REMOVED*** addAlgorithmsToStagingHost(algoSet, hostname, uuid, /*updateSVN,*/test);***REMOVED*** -***REMOVED*** ***REMOVED*** catch (IOException e) { -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** catch (InterruptedException e) { -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** catch (SVNException e) { -***REMOVED*** ***REMOVED*** TODO Auto-generated catch block -***REMOVED*** e.printStackTrace(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED***).start(); -***REMOVED*** ***REMOVED*** this line will execute immediately, not waiting for your task to -***REMOVED*** ***REMOVED*** complete -***REMOVED*** System.out.println(uuid); -***REMOVED*** return uuid; -***REMOVED******REMOVED*** - - - public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException { - - File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/" - + a + File.separator + "logs"); - path.mkdirs(); - File n = new File(path + File.separator + a); - ***REMOVED*** String addr = InetAddress.getLocalHost().getHostAddress(); - - return new File(n.getPath()).toURI().toURL(); -***REMOVED*** - -***REMOVED*** public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, /*boolean updateSVN,*/ boolean test) throws IOException, InterruptedException, SVNException { -***REMOVED*** -***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre) -***REMOVED*** Cluster cluster = new Cluster(); -***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) { -***REMOVED*** ***REMOVED***for (Host h : new ISClient().listDataminersInVRE()) { -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** ***REMOVED*** apply the changes -***REMOVED*** AnsibleBridge a = new AnsibleBridge(); -***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/ test).getWorkerId(); -***REMOVED*** -***REMOVED******REMOVED*** - -***REMOVED*** public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) -***REMOVED*** throws IOException, InterruptedException, SVNException { -***REMOVED*** -***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre) -***REMOVED*** Cluster cluster = new Cluster(); -***REMOVED*** for (Host h : new HAProxy().listDataMinersByCluster()) { -***REMOVED*** if (h.getName().equals(hostname)) { -***REMOVED*** cluster.addHost(h); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** if(ISClient.getHProxy().equals(hostname)){ -***REMOVED*** ***REMOVED*** cluster.addHost(new ISClient().getDataminer(hostname)); -***REMOVED*** ***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** apply the changes -***REMOVED*** AnsibleBridge a = new AnsibleBridge(); -***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); -***REMOVED*** -***REMOVED******REMOVED*** - -***REMOVED*** -***REMOVED*** public String addAlgorithmsToStagingHost(AlgorithmSet algorithms, String hostname, String uuid, /*boolean updateSVN,*/boolean test) -***REMOVED*** throws IOException, InterruptedException, SVNException { -***REMOVED*** Cluster cluster = new Cluster(); -***REMOVED*** Host h = new Host(); -***REMOVED*** h.setName(hostname); -***REMOVED*** cluster.addHost(h); -***REMOVED*** -***REMOVED*** AnsibleBridge a = new AnsibleBridge(); -***REMOVED*** return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); -***REMOVED*** -***REMOVED******REMOVED*** - - ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java new file mode 100644 index 0000000..3bacdc3 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -0,0 +1,33 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.service; + +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; + +public class ProductionPublishingJob extends DMPMJob { + + private Algorithm algorithm; + private Cluster prodCluster; + + public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster) { + super(svnUpdater); + this.algorithm = algorithm; + this.prodCluster = prodCluster; +***REMOVED*** + + @Override + protected void execute() { + try { + this.updateSVNDependencies(this.algorithm, false); + + int ret = this.executeAnsibleWorker( + createWorker(this.algorithm, this.prodCluster, false, "gcube")); + + ***REMOVED*** catch (Exception e) { + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java new file mode 100644 index 0000000..e874ff9 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -0,0 +1,46 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.service; + +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; + +public class StagingJob extends DMPMJob { + + + private Algorithm algorithm; + private Cluster stagingCluster; + private Cluster rProtoCluster; + + public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster) { + super(svnUpdater); + this.algorithm = algorithm; + this.stagingCluster = stagingCluster; + this.rProtoCluster = rProtoCluster; +***REMOVED*** + + @Override + protected void execute() { + try { + + int ret = this.executeAnsibleWorker( + createWorker(this.algorithm, this.stagingCluster, true, "root")); + + if(ret == 0){ + + + this.updateSVNDependencies(this.algorithm, true); + + + int ret2 = this.executeAnsibleWorker( + createWorker(this.algorithm, this.rProtoCluster, false, "gcube")); + + ***REMOVED*** + + + + ***REMOVED*** catch (Exception e) { + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java index 3299c09..70af35f 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java @@ -11,13 +11,15 @@ import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackagePars public class AlgorithmBuilder { - public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException { - return create(algorithmPackageURL, null, null, null, null, null, null, null); + public static Algorithm create(String algorithmPackageURL, String category) throws IOException, InterruptedException { + return create(algorithmPackageURL, null, null, null, null, category, null, null); ***REMOVED*** public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description, String category, String algorithmType, String skipJava) throws IOException, InterruptedException { - Algorithm algo = new AlgorithmPackageParser().parsePackage(algorithmPackageURL); + + + Algorithm algo = new AlgorithmPackageParser().parsePackage(algorithmPackageURL); if(category != null){ diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java new file mode 100644 index 0000000..65ca353 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java @@ -0,0 +1,44 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +***REMOVED*** + +***REMOVED*** +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; +***REMOVED*** +***REMOVED*** + +public class ClusterBuilder { + + + ***REMOVED***1. to complete + public static Cluster getStagingDataminerCluster(){ + Cluster cluster = new Cluster(); + Props p = new Props(); + Host h = new Host(p.getStagingHost()); + ***REMOVED***TODO: read this from configuration or IS? + h.setName(p.getStagingHost()); + cluster.addHost(h); + return cluster; +***REMOVED*** + + public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException{ + Cluster cluster = new Cluster(); + for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) { + cluster.addHost(h); + ***REMOVED*** + return cluster; +***REMOVED*** + + public static Cluster getRProtoCluster() throws IOException{ + ***REMOVED***Assumes the service is running in RPrototypingLab + + String token = SecurityTokenProvider.instance.get(); + String targetVRE = ScopeProvider.instance.get(); + + return getVRECluster(token, targetVRE); +***REMOVED*** + + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index 21c8a90..2711d59 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -25,9 +25,8 @@ public class SVNUpdater { private SVNRepository svnRepository; public SVNUpdater(String rootURL) throws SVNException { - final SVNRepository svnRepository = this.getSvnRepository( + this.svnRepository = this.getSvnRepository( rootURL); - ***REMOVED*** @@ -35,7 +34,7 @@ public class SVNUpdater { SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); repository.setAuthenticationManager(authManager); - ***REMOVED***System.out.println(repository.getLocation()); + System.out.println(repository.getLocation()); return repository; ***REMOVED*** @@ -43,7 +42,7 @@ public class SVNUpdater { - public List updateSVN(String file, List ldep) throws SVNException, IOException { + public void updateSVN(String file, List ldep) { try { final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); @@ -73,9 +72,13 @@ public class SVNUpdater { new ByteArrayInputStream(bytes), commitEditor, true); commitEditor.closeFile(file, checksum); commitEditor.closeEdit(); - return aa; + + + ***REMOVED***catch(Exception ex){ + ex.printStackTrace(); + ***REMOVED*** - ***REMOVED*** finally { + finally { svnRepository.closeSession(); ***REMOVED*** ***REMOVED*** diff --git a/src/main/resources/service.properties b/src/main/resources/service.properties index aa0b8ea..2011784 100644 --- a/src/main/resources/service.properties +++ b/src/main/resources/service.properties @@ -1,4 +1,4 @@ #YML node file -STAGING_HOST: dataminer1-devnext.d4science.org +STAGING_HOST: dataminer-proto-ghost.d4science.org SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ #HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0 diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java new file mode 100644 index 0000000..4a87a4c --- /dev/null +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java @@ -0,0 +1,46 @@ +package org.gcube.dataanalysis.dataminerpoolmanager; + +***REMOVED*** + +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.service.DMPMJob; +import org.gcube.dataanalysis.dataminer.poolmanager.service.ProductionPublishingJob; +import org.gcube.dataanalysis.dataminer.poolmanager.service.StagingJob; +import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.tmatesoft.svn.core.SVNException; + +public class JobTest { + + public static void main(String[] args) throws SVNException, IOException, InterruptedException{ + + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + + + + SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); + + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); + + + Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); + + + Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); + + + DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster); + + job.start(); + + +***REMOVED*** Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); +***REMOVED*** +***REMOVED*** DMPMJob job2 = new ProductionPublishingJob(svnUpdater, algo, prodCluster); + +***REMOVED*** + +***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java new file mode 100644 index 0000000..aa15ba5 --- /dev/null +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java @@ -0,0 +1,27 @@ +package org.gcube.dataanalysis.dataminerpoolmanager; + +***REMOVED*** +***REMOVED*** +***REMOVED*** + +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.tmatesoft.svn.core.SVNException; + +public class SVNUpdaterTest { + + public static void main(String[] args) throws SVNException, IOException { + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + + + + SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); + + String test = "testDep"; + List ldep = new LinkedList<>(); + ldep.add(test); + + svnUpdater.updateSVN("test_r_cran_pkgs.txt", ldep); +***REMOVED*** + +***REMOVED***