diff --git a/.classpath b/.classpath deleted file mode 100644 index 70eaec6..0000000 --- a/.classpath +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.project b/.project deleted file mode 100644 index 367d621..0000000 --- a/.project +++ /dev/null @@ -1,42 +0,0 @@ - - - dataminer-pool-manager - - - - - - org.eclipse.wst.jsdt.core.javascriptValidator - - - - - org.eclipse.wst.common.project.facet.core.builder - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.wst.validation.validationbuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jem.workbench.JavaEMFNature - org.eclipse.wst.common.modulecore.ModuleCoreNature - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - org.eclipse.wst.common.project.facet.core.nature - org.eclipse.wst.jsdt.core.jsNature - - diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index 2b6660a..efc0677 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -116,9 +116,32 @@ public class AnsibleWorker { ***REMOVED*** Serialize role in the workdir AnsibleSerializeHelper.serializeRole(r, this.getRolesDir()); ***REMOVED*** - - - + + + + public int execute(PrintStream ps) + throws IOException, InterruptedException, SVNException { + + System.out.println(this.getWorkdir()); + try { + Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + + " " + this.getPlaybookFile().getAbsolutePath()); + + inheritIO(p.getInputStream(), ps); + inheritIO(p.getErrorStream(), ps); + + + return p.waitFor(); + + ***REMOVED*** catch (IOException e) { + e.printStackTrace(); + ***REMOVED*** + + return -1; +***REMOVED*** + + + public void apply(AlgorithmSet as, PrintStream ps, boolean test) throws IOException, InterruptedException, SVNException { ***REMOVED*** TODO execute the playbook and return output diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index b5a4096..00cdc26 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -89,9 +89,55 @@ public class AnsibleBridge { - public AnsibleWorker createWorker() { + public AnsibleWorker createWorker(Algorithm algorithm, Cluster dataminerCluster, boolean includeAlgorithmDependencies, String user) throws IOException { File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); AnsibleWorker worker = new AnsibleWorker(workerRoot); + + List algoRoles = new Vector<>(); + + ***REMOVED*** add algorithms and dependencies to the worker + for (Role r : this.generateRoles(algorithm)) { + algoRoles.add(r); + worker.addRole(r); + ***REMOVED*** + + ***REMOVED***to comment the for in case of just install algo + if(includeAlgorithmDependencies){ + for (Dependency d : algorithm.getDependencies()) { + for (Role r : this.generateRoles(d)) { + worker.addRole(r); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + + + ***REMOVED*** add static roles + for(Role r:this.getStaticRoleManager().getStaticRoles()) { + worker.addRole(r); + ***REMOVED*** + + ***REMOVED*** generate the inventory + Inventory inventory = new Inventory(); + for (Host h : dataminerCluster.getHosts()) { + AnsibleHost ah = new AnsibleHost(h.getName()); + inventory.addHost(ah, "universe"); + inventory.addHost(ah, "d4science"); + ***REMOVED*** + worker.setInventory(inventory); + + ***REMOVED*** generate the playbook + Playbook playbook = new Playbook(); + playbook.setRemote_user(user); + playbook.applyTo("universe"); + for(Role r:algoRoles) { + ***REMOVED*** add only 'add' roles + if(!r.getName().endsWith("remove")) { + playbook.addRole(r.getName()); + ***REMOVED*** + ***REMOVED*** + + worker.setPlaybook(playbook); + return worker; ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index 60a2c6b..600a3be 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -42,7 +42,12 @@ public interface PoolManager { String addAlgorithmToVRE(Algorithm algo, String vre, boolean test ) throws IOException, InterruptedException; String addAlgorithmToHost(Algorithm algo, String host,boolean test) throws IOException, InterruptedException; - + + + String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException; + String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException; + + Algorithm extractAlgorithm(String url) throws IOException; String getLogById(String logId) throws IOException; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index fbd63a2..e8a00b9 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -1,281 +1,52 @@ -***REMOVED***package org.gcube.dataanalysis.dataminer.poolmanager.rest; -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED***import java.net.MalformedURLException; -***REMOVED***import java.net.ProxySelector; -***REMOVED******REMOVED*** -***REMOVED***import java.net.UnknownHostException; -***REMOVED*** -***REMOVED***import javax.ws.rs.GET; -***REMOVED***import javax.ws.rs.Path; -***REMOVED***import javax.ws.rs.Produces; -***REMOVED***import javax.ws.rs.QueryParam; -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -***REMOVED***import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; -***REMOVED******REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED***@Path("/") -***REMOVED***public class RestPoolManager implements PoolManager { -***REMOVED*** -***REMOVED*** private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class); -***REMOVED*** -***REMOVED*** private PoolManager service = new DataminerPoolManager(); -***REMOVED*** -***REMOVED*** -***REMOVED*** @GET -***REMOVED*** @Path("/add") -***REMOVED*** @Produces("text/plain") -***REMOVED*** public String addAlgorithmToVRE( -***REMOVED*** @QueryParam("algorithm") String algorithm, -***REMOVED*** @QueryParam("vre") String vre, -***REMOVED*** @QueryParam("name") String name, -***REMOVED*** @QueryParam("description") String description, -***REMOVED*** @QueryParam("category") String category, -***REMOVED*** @QueryParam("algorithmType") String algorithmType, -***REMOVED*** @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre); -***REMOVED*** Algorithm algo = service.extractAlgorithm(algorithm); -***REMOVED*** -***REMOVED*** if (algo.getCategory() == null){ -***REMOVED*** algo.setCategory(category); -***REMOVED*** ***REMOVED*** else category = algo.getCategory(); -***REMOVED*** -***REMOVED*** if (algo.getAlgorithmType() == null){ -***REMOVED*** algo.setAlgorithmType(algorithmType); -***REMOVED*** ***REMOVED*** else algorithmType = algo.getCategory(); -***REMOVED*** -***REMOVED*** if (algo.getSkipJava() == null){ -***REMOVED*** algo.setSkipJava(skipJava); -***REMOVED*** ***REMOVED*** else skipJava = algo.getSkipJava(); -***REMOVED*** -***REMOVED*** if (algo.getName() == null){ -***REMOVED*** algo.setCategory(name); -***REMOVED*** ***REMOVED*** else name = algo.getName(); -***REMOVED*** -***REMOVED*** if (algo.getDescription() == null){ -***REMOVED*** algo.setDescription(description);; -***REMOVED*** ***REMOVED*** else description = algo.getDescription(); -***REMOVED*** -***REMOVED*** return service.addAlgorithmToVRE(algo, vre); -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** @GET -***REMOVED*** @Path("/log") -***REMOVED*** @Produces("text/plain") -***REMOVED*** public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** LOGGER.debug("Returning Log =" + logUrl); -***REMOVED*** return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl)); -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** @Override -***REMOVED*** public Algorithm extractAlgorithm(String url) throws IOException { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** return null; -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** public static void main(String[] args) throws IOException, InterruptedException { -***REMOVED*** RestPoolManager a = new RestPoolManager(); -***REMOVED*** ***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); -***REMOVED*** -***REMOVED*** -***REMOVED*** ***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ScopeProvider.instance.set("/gcube/devNext/NextNext"); -***REMOVED*** a.addAlgorithmToVRE( -***REMOVED*** "http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0", -***REMOVED*** "/gcube/devNext/NextNext", -***REMOVED*** null, null, "test", "transducerers", "N"); -***REMOVED*** -***REMOVED*** ***REMOVED***System.out.println(a.getLogById("34ac474d-b9df-4929-87e1-2a0ae26cf898")); -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** @Override -***REMOVED*** public void getLogId(Algorithm algo, String vre) { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** @Override -***REMOVED*** public String getScriptFromURL(URL logId) throws IOException { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** return null; -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** @Override -***REMOVED*** public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** return null; -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** @Override -***REMOVED*** public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { -***REMOVED*** ***REMOVED*** TODO Auto-generated method stub -***REMOVED*** return null; -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED******REMOVED*** package org.gcube.dataanalysis.dataminer.poolmanager.rest; -import java.awt.color.ICC_ColorSpace; -***REMOVED*** -import java.net.MalformedURLException; -import java.net.ProxySelector; -***REMOVED*** -import java.net.UnknownHostException; ***REMOVED*** ***REMOVED*** -import java.util.Random; -import java.util.Set; -import java.util.UUID; - -import javax.ws.rs.DefaultValue; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; - -import org.apache.commons.lang.math.RandomUtils; -***REMOVED*** -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; +import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; ***REMOVED*** ***REMOVED*** import org.tmatesoft.svn.core.SVNException; -import ch.qos.logback.core.subst.Token; +import javax.ws.rs.*; +***REMOVED*** +import java.net.MalformedURLException; +***REMOVED*** +import java.net.UnknownHostException; +***REMOVED*** +import java.util.Set; @Path("/") public class RestPoolManager implements PoolManager { private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class); - private PoolManager service = new DataminerPoolManager(); + private DataminerPoolManager service = new DataminerPoolManager(); + + + @GET + @Path("/algorithm/stage") + @Produces("text/plain") + public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException { + Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); + return this.service.stageAlgorithm(algo); +***REMOVED*** + + + @GET + @Path("/algorithm/add") + @Produces("text/plain") + public String publishAlgorithm(String algorithmPackageURL, String targetVREToken) throws IOException, InterruptedException { + Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); + return this.service.publishAlgorithm(algo, targetVREToken); +***REMOVED*** /* * /scopes/ POST ***REMOVED*** add an algorithm to all dataminers in the scope * /hosts/ POST ***REMOVED*** add an algorithm to the given host */ - @GET - @Path("/scopes/add") - @Produces("text/plain") - public String addAlgorithmToVRE( - @QueryParam("algorithm") String algorithm, - @QueryParam("name") String name, - @QueryParam("description") String description, - @QueryParam("category") String category, - @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType, - @DefaultValue("N") @QueryParam("skipJava") String skipJava, - @DefaultValue("false") @QueryParam("publish") boolean publish, - @DefaultValue("false") @QueryParam("updateSVN") boolean updateSVN) - throws IOException, InterruptedException, SVNException { - Algorithm algo = this.getAlgorithm(algorithm, /*vre*/null, null, name, description, category, algorithmType, skipJava); - ***REMOVED*** publish algo - if (publish) { - service.addAlgToIs(algo); - ***REMOVED*** - return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),/*updateSVN,*/false); -***REMOVED*** - - ***REMOVED***Test Phase - ***REMOVED***The DMPM contacts the Staging Dataminer and installs the algorithm and the dependencies - @GET - @Path("/algo/add") - @Produces("text/plain") - public String addAlgorithmToStagingHost( - @QueryParam("algorithm") String algorithm, - @DefaultValue("insert hostname here") @QueryParam("hostname") String hostname, - @QueryParam("name") String name, - @QueryParam("description") String description, - @QueryParam("category") String category, - @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType, - @DefaultValue("N") @QueryParam("skipJava") String skipJava, - @DefaultValue("prodtoken") @QueryParam("prodtoken") String prodtoken, - @DefaultValue("true") @QueryParam("test") boolean test) - throws IOException, InterruptedException, SVNException { - String id=""; - Algorithm algo = this.getAlgorithm(algorithm, ScopeProvider.instance.get(), hostname, name, description, category, algorithmType,skipJava); - - ***REMOVED***test phase - if(test){ - service.addAlgorithmToHost(algo, hostname,true); - id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),true); - ***REMOVED*** - ***REMOVED***release phase - if(!test){ - SecurityTokenProvider.instance.set(prodtoken); - id = service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),false); - ***REMOVED*** - - return id; - -***REMOVED*** - - - private Algorithm getAlgorithm(String algorithm, String vre, String hostname, String name, String description, - String category, String algorithmType, String skipJava) throws IOException, InterruptedException { - Algorithm algo = service.extractAlgorithm(algorithm); - - if (algo.getCategory() == null) { - algo.setCategory(category); - ***REMOVED*** else - algo.setCategory(algo.getCategory()); - - if (algo.getAlgorithmType() == null) { - algo.setAlgorithmType(algorithmType); - ***REMOVED*** else - algo.setAlgorithmType(algo.getCategory()); - - if (algo.getSkipJava() == null) { - algo.setSkipJava(skipJava); - ***REMOVED*** else - algo.setSkipJava(algo.getSkipJava()); - - if (algo.getName() == null) { - algo.setName(name); - ***REMOVED*** else - algo.setName(algo.getName()); - - if (algo.getDescription() == null) { - algo.setDescription(description); - ; - ***REMOVED*** else - algo.setDescription(algo.getDescription()); - - return algo; -***REMOVED*** - @GET @Path("/log") @Produces("text/plain") @@ -304,40 +75,6 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3")); ***REMOVED*** System.out.println(aa.getAlgoFromIs()); - - RestPoolManager a = new RestPoolManager(); - a.addAlgorithmToVRE( - "http:***REMOVED***data.d4science.org/TVc0TW9Ud1FjYlppK0NHd2pvU0owNmRFWHE4OW4xSGNHbWJQNStIS0N6Yz0", - "test", - null, - null, - "N", - "transducerers", - false, - false); - -***REMOVED*** a.addAlgorithmToHost( -***REMOVED*** "http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", -***REMOVED*** "dataminer1-pre.d4science.org", -***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", -***REMOVED*** "N",false, false); - RestPoolManager b = new RestPoolManager(); - - b.addAlgorithmToStagingHost( - "http:***REMOVED***data.d4science.org/TVc0TW9Ud1FjYlppK0NHd2pvU0owNmRFWHE4OW4xSGNHbWJQNStIS0N6Yz0", - "dataminer1-devnext.d4science.org", - null, - null, - "N", - "transducerers", - false, - false -***REMOVED*** PoolManager aa = new DataminerPoolManager(); -***REMOVED*** List ls = new LinkedList(); -***REMOVED*** String afa = "test"; -***REMOVED*** ls.add(afa); -***REMOVED*** -***REMOVED*** System.out.println(aa.updateSVN("r_deb_pkgs.txt", ls)); ***REMOVED*** @Override diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index c5d3e58..cc895b0 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -1,68 +1,142 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - ***REMOVED*** -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -***REMOVED*** -***REMOVED*** -import java.io.StringWriter; -import java.net.MalformedURLException; -***REMOVED*** -import java.net.URLConnection; -import java.net.UnknownHostException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -***REMOVED*** -import java.util.Set; -import java.util.SortedSet; -import java.util.UUID; - import org.gcube.common.resources.gcore.GenericResource; -import org.gcube.common.resources.gcore.HostingNode; import org.gcube.common.resources.gcore.Resources; -import org.gcube.common.resources.gcore.Software.Profile.Dependency; ***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -***REMOVED*** -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*; import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; import org.gcube.informationsystem.publisher.RegistryPublisherFactory; import org.gcube.informationsystem.publisher.ScopedPublisher; import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; - ***REMOVED*** ***REMOVED*** -import org.tmatesoft.svn.core.SVNCommitInfo; import org.tmatesoft.svn.core.SVNException; -import org.tmatesoft.svn.core.SVNURL; -import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; -import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; -import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream; -import org.tmatesoft.svn.core.io.ISVNEditor; -import org.tmatesoft.svn.core.io.SVNRepository; -import org.tmatesoft.svn.core.io.SVNRepositoryFactory; -import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator; -import org.tmatesoft.svn.core.wc.SVNWCUtil; -public class DataminerPoolManager implements PoolManager { +import java.io.*; +import java.net.MalformedURLException; +***REMOVED*** +import java.net.URLConnection; +import java.net.UnknownHostException; +import java.util.*; + +import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; +import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; + +public class DataminerPoolManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); + + public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException { + + Cluster cluster = new Cluster(); + Host h = new Host(); + h.setName(getStagingDataminerHostname()); + cluster.addHost(h); + + ***REMOVED***Assumes the service is running in RPrototypingLab + String token = SecurityTokenProvider.instance.get(); + + return addAlgorithmToCluster(algo, cluster, true, "root", true, token); +***REMOVED*** + + public String publishAlgorithm(Algorithm algo, String targetVREToken) throws IOException, InterruptedException { + Cluster cluster = new Cluster(); + for (Host h : new HAProxy().listDataMinersByCluster()) { + cluster.addHost(h); + ***REMOVED*** + return addAlgorithmToCluster(algo, cluster, false, "gcube", false, targetVREToken); + +***REMOVED*** + + + private String getStagingDataminerHostname(){ + return null; +***REMOVED*** + + private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException { + for (Dependency d : algo.getDependencies()) { + + if (d.getType().equals("os")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("cran")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("github")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls); + ***REMOVED*** + ***REMOVED*** +***REMOVED*** + + + private void createISResource(Algorithm algo, String vreToken){ + ***REMOVED***TODO: implement method + + ***REMOVED***TODO: create the resource only if not already present +***REMOVED*** + + + + /** + * + * @param algo + * @param dataminerCluster + * @return uuid of the execution + */ + public String addAlgorithmToCluster( + final Algorithm algo, + Cluster dataminerCluster, + boolean includeAlgorithmDependencies, + String user, + final boolean stagingVRE, + final String targetVREToken) throws IOException { + + AnsibleBridge ansibleBridge = new AnsibleBridge(); + final AnsibleWorker worker = ansibleBridge.createWorker(algo, dataminerCluster, includeAlgorithmDependencies, user); + + new Thread(new Runnable() { + @Override + public void run() { + try { + File path = new File(worker.getWorkdir() + File.separator + "logs"); + path.mkdirs(); + File n = new File(path + File.separator + worker.getWorkerId()); + FileOutputStream fos = new FileOutputStream(n); + PrintStream ps = new PrintStream(fos); + int retValue = worker.execute(ps); + System.out.println("Log stored to to " + n.getAbsolutePath()); + + if(retValue == 0) { + updateSVNDependencies(algo, stagingVRE); + createISResource(algo, targetVREToken); + ***REMOVED*** + ***REMOVED*** destroy the worker + worker.destroy(); + ***REMOVED*** catch (Exception e) { + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** + ***REMOVED***).start(); + + ***REMOVED*** this line will execute immediately, not waiting for task to + ***REMOVED*** complete + return worker.getWorkerId(); +***REMOVED*** + + public String getScriptFromURL(URL url) throws IOException { if (url == null) { @@ -152,6 +226,7 @@ public class DataminerPoolManager implements PoolManager { return uuid; ***REMOVED*** + public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException { File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/" @@ -209,23 +284,7 @@ public class DataminerPoolManager implements PoolManager { return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, /*updateSVN,*/test).getWorkerId(); ***REMOVED*** - - public Algorithm extractAlgorithm(String url) throws IOException { - return new AlgorithmPackageParser().parsePackage(url); -***REMOVED*** - - @Override - public void getLogId(Algorithm algo, String vre) { - ***REMOVED*** TODO Auto-generated method stub - -***REMOVED*** - - @Override - public String getLogById(String logId) throws IOException { - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** ***REMOVED*** 2017 March 29 @@ -256,7 +315,6 @@ public class DataminerPoolManager implements PoolManager { ***REMOVED*** ***REMOVED*** - @Override public void addAlgToIs(Algorithm algo) { GenericResource a = new GenericResource(); a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); @@ -317,16 +375,6 @@ public class DataminerPoolManager implements PoolManager { ***REMOVED*** ***REMOVED*** - ***REMOVED*** public Algorithm getAlgoById(String id) { - ***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) { - ***REMOVED*** if (aa.getId().equals(id)) { - ***REMOVED*** return aa; - ***REMOVED*** ***REMOVED*** - ***REMOVED*** ***REMOVED*** - ***REMOVED*** return null; - ***REMOVED*** ***REMOVED*** - - @Override public Set getAlgoFromIs() { ***REMOVED*** TODO Auto-generated method stub @@ -343,8 +391,6 @@ public class DataminerPoolManager implements PoolManager { ***REMOVED*** - - @Override public List updateSVN(String file, List ldep) throws SVNException, IOException { ***REMOVED*** TODO Auto-generated method stub return null; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java new file mode 100644 index 0000000..0e1f345 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/AlgorithmBuilder.java @@ -0,0 +1,52 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; + +***REMOVED*** + +/** + * Created by ggiammat on 5/9/17. + */ +public class AlgorithmBuilder { + + + public static Algorithm create(String algorithmPackageURL) throws IOException, InterruptedException { + return create(algorithmPackageURL, null, null, null, null, null, null, null); + ***REMOVED*** + + public static Algorithm create(String algorithmPackageURL, String vre, String hostname, String name, String description, + String category, String algorithmType, String skipJava) throws IOException, InterruptedException { + Algorithm algo = new AlgorithmPackageParser().parsePackage(algorithmPackageURL); + + + if(category != null){ + algo.setCategory(category); + ***REMOVED*** + + ***REMOVED***FIXME: do the same done for category + if (algo.getAlgorithmType() == null) { + algo.setAlgorithmType(algorithmType); + ***REMOVED*** else + algo.setAlgorithmType(algo.getCategory()); + + if (algo.getSkipJava() == null) { + algo.setSkipJava(skipJava); + ***REMOVED*** else + algo.setSkipJava(algo.getSkipJava()); + + if (algo.getName() == null) { + algo.setName(name); + ***REMOVED*** else + algo.setName(algo.getName()); + + if (algo.getDescription() == null) { + algo.setDescription(description); + ; + ***REMOVED*** else + algo.setDescription(algo.getDescription()); + + return algo; + ***REMOVED*** + +***REMOVED***