diff --git a/.classpath b/.classpath index 0cad5db..e50289f 100755 --- a/.classpath +++ b/.classpath @@ -17,6 +17,7 @@ + diff --git a/.settings/org.eclipse.wst.common.component b/.settings/org.eclipse.wst.common.component index 6bfbbcc..0da405d 100755 --- a/.settings/org.eclipse.wst.common.component +++ b/.settings/org.eclipse.wst.common.component @@ -4,6 +4,7 @@ + diff --git a/src/test/java/AlgorithmPackageParserTest.java b/src/test/java/AlgorithmPackageParserTest.java deleted file mode 100644 index c728ac2..0000000 --- a/src/test/java/AlgorithmPackageParserTest.java +++ /dev/null @@ -1,117 +0,0 @@ - - -***REMOVED*** -import java.io.InputStream; -***REMOVED*** -***REMOVED*** -import java.util.Vector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; - -public class AlgorithmPackageParserTest ***REMOVED*** - - private static int BUFFER_SIZE = 2048; - - public void extractAllAlgorithms() throws IOException ***REMOVED*** - String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; - List commands = this.extractAddAlgorithmCommands(url); - AlgorithmSet algorithms = new AlgorithmSet(); - for (String cmd : commands) ***REMOVED*** - System.out.println("-----------------------------------------"); - System.out.println(cmd); - AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - System.out.println(aac); - - ***REMOVED*** start creating the algo from the command - Algorithm algo = new Algorithm(); - algo.setAlgorithmType(aac.getAlgorithmType()); - algo.setCategory(aac.getCategory()); - algo.setClazz(aac.getClazz()); - algo.setDescription(aac.getDescription()); - algo.setName(aac.getName()); - algo.setPackageURL(aac.getUrl()); - algo.setSkipJava(aac.getSkipjava()); - - ***REMOVED*** then override with info from the package - if (aac.getUrl().length() > 4) ***REMOVED*** - Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); - if (packagedAlgo != null) ***REMOVED*** - algo.setDependencies(packagedAlgo.getDependencies()); - ***REMOVED*** - ***REMOVED*** - algorithms.addAlgorithm(algo); - break; - ***REMOVED*** - ***REMOVED***to uncomment - ***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - ***REMOVED*** "/gcube/devNext/NextNext"); - ***REMOVED*** - - /** - * Extract 'addAlgorithm' commands from a file containing wiki-table-style - * entries for algorithm. - * - * @return - * @throws IOException - */ - private List extractAddAlgorithmCommands(String listUrl) - throws IOException ***REMOVED*** - URL url = new URL(listUrl); - InputStream is = url.openStream(); - - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = is.read(buffer)) != -1) ***REMOVED*** - s.append(new String(buffer, 0, read)); - ***REMOVED*** - List out = new Vector<>(); - String[] lines = s.toString().split("\n"); - for (String line : lines) ***REMOVED*** - System.out.println("--------------------"); - if (!line.isEmpty()) ***REMOVED*** - String[] parts = line.split("\\|"); - int c = 1; - for (String part : parts) ***REMOVED*** - if (part == null || part.trim().isEmpty()) ***REMOVED*** - continue; - ***REMOVED*** - System.out.println(c + ". " + part); - c++; - if (part.contains("addAlgorithm.sh")) ***REMOVED*** - String cmd = part.trim(); - cmd = cmd.replaceAll("", ""); - cmd = cmd.replaceAll("", ""); - System.out.println(cmd); - ***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - ***REMOVED*** System.out.println(aac); - out.add(cmd); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - /** - * Create an Algorithm starting from the algorithm jar. - * - * @param url - * @return - * @throws IOException - */ - private Algorithm extractAlgorithm(String url) throws IOException ***REMOVED*** - return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** - - public static void main(String[] args) throws Exception ***REMOVED*** -***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector( -***REMOVED*** "/home/ngalante/.proxy-settings")); - - new AlgorithmPackageParserTest().extractAllAlgorithms(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/AnsibleWorkerTest.java b/src/test/java/AnsibleWorkerTest.java deleted file mode 100644 index 664df42..0000000 --- a/src/test/java/AnsibleWorkerTest.java +++ /dev/null @@ -1,33 +0,0 @@ - - -import java.io.File; -***REMOVED*** -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleWorkerTest ***REMOVED*** - - public static void main(String[] args) throws IOException ***REMOVED*** - AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString())); - - System.out.println("created worker named " + worker.getWorkerId()); - - worker.setInventory(new Inventory()); - worker.setPlaybook(new Playbook()); - - Role r = new Role(); - r.setName("latex"); - - RoleFile tf = new RoleFile("main", "do something special for " + r.getName()); - r.addTaskFile(tf); - worker.addRole(r); - - ***REMOVED***worker.apply(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/DataminerPoolManagerTest.java b/src/test/java/DataminerPoolManagerTest.java deleted file mode 100644 index ad70c13..0000000 --- a/src/test/java/DataminerPoolManagerTest.java +++ /dev/null @@ -1,225 +0,0 @@ - - -import java.net.ProxySelector; - -***REMOVED*** -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class DataminerPoolManagerTest ***REMOVED*** - - /* - private static Map domains = new HashMap<>(); - private static Map clusters = new HashMap<>(); - - private static Dependency createDependency(String depName) ***REMOVED*** - String[] parts = depName.split(":"); - Dependency out = new Dependency(); - if(parts.length>1) ***REMOVED*** - out.setType(parts[0]); - out.setName(parts[1]); - ***REMOVED*** else ***REMOVED*** - out.setType("os"); - out.setName(depName); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Algorithm createAlgorithm(String name, String ... deps) ***REMOVED*** - Algorithm a = new Algorithm(); - a.setName(name); - for(String dep:deps) ***REMOVED*** - a.addDependency(createDependency(dep)); - ***REMOVED*** - return a; - ***REMOVED*** - - private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) ***REMOVED*** - AlgorithmSet out = new AlgorithmSet(); - out.setName(name); - for(Algorithm a:algs) ***REMOVED*** - out.addAlgorithm(a); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Domain getDomain(String name) ***REMOVED*** - if(domains.get(name)==null) ***REMOVED*** - Domain d = new Domain(); - d.setName(name); - domains.put(name, d); - return d; - ***REMOVED*** else ***REMOVED*** - return domains.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Host createHost(String hostname, String domainName) ***REMOVED*** - Host out = new Host(); - out.setName(hostname); - Domain d = getDomain(domainName); - out.setDomain(d); - return out; - ***REMOVED*** - - private static Cluster getCluster(String name) ***REMOVED*** - if(clusters.get(name)==null) ***REMOVED*** - Cluster d = new Cluster(); - d.setName(name); - clusters.put(name, d); - return d; - ***REMOVED*** else ***REMOVED*** - return clusters.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Collection extractDependencies() ***REMOVED*** - Collection out = new TreeSet<>(new DependencyComparator()); - for(Cluster c:clusters.values()) ***REMOVED*** - for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED*** - for(Algorithm a:as.getAlgorithms()) ***REMOVED*** - for(Dependency d:a.getDependencies()) ***REMOVED*** - out.add(d); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - private static Collection extractAlgorithms() ***REMOVED*** - Collection out = new TreeSet<>(new AlgorithmComparator()); - for(Cluster c:clusters.values()) ***REMOVED*** - for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED*** - for(Algorithm a:as.getAlgorithms()) ***REMOVED*** - out.add(a); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - static ***REMOVED*** - - Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package"); - Algorithm ensemble = createAlgorithm("ensemble", "python"); - Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah"); - - AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe); - AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble); - AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe); - - Cluster cluster1 = getCluster("cluster-1"); - cluster1.addHost(createHost("host1", "domain1")); - cluster1.addHost(createHost("host2", "domain1")); - cluster1.addHost(createHost("host3", "domain1")); - cluster1.addHost(createHost("host1", "domain2")); - cluster1.addHost(createHost("host2", "domain2")); - - Cluster cluster2 = getCluster("cluster-2"); - cluster2.addHost(createHost("host4", "domain1")); - cluster2.addHost(createHost("host5", "domain1")); - cluster2.addHost(createHost("host6", "domain1")); - cluster2.addHost(createHost("host3", "domain2")); - cluster2.addHost(createHost("host4", "domain2")); - cluster2.addHost(createHost("host5", "domain2")); - - cluster1.addAlgorithmSet(as1); - cluster1.addAlgorithmSet(as2); - - cluster2.addAlgorithmSet(as1); - cluster2.addAlgorithmSet(as3); - - ***REMOVED*** - */ - - public static void main(String[] args) throws Exception ***REMOVED*** - /* - AnsibleBridge ab = new AnsibleBridge(); - ab.printInventoryByDomainAndSets(clusters.values()); - System.out.println("-----------"); - ab.printInventoryBySets(clusters.values()); - - AnsibleWorker worker = ab.createWorker(); - - for(Algorithm a:extractAlgorithms()) ***REMOVED*** - for(Role r:ab.generateRoles(a)) ***REMOVED*** - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - - for(Dependency d:extractDependencies()) ***REMOVED*** - for(Role r:ab.generateRoles(d)) ***REMOVED*** - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - */ - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm algorithm = new Algorithm(); -***REMOVED*** algorithm.setName("ichtyop"); -***REMOVED*** algorithm.setClazz("org.gcube..."); -***REMOVED*** algorithm.setDescription("some description"); -***REMOVED*** -***REMOVED*** Dependency d = new Dependency(); -***REMOVED*** d.setName("libpng"); -***REMOVED*** d.setType("os"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-other-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm ewe = new Algorithm(); -***REMOVED*** ewe.setName("ewe"); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** ewe.addDependency(d); - - AlgorithmSet algorithms = new AlgorithmSet(); - algorithms.setName("dummy-set"); - - -***REMOVED*** algorithms.addAlgorithm(algorithm); -***REMOVED*** algorithms.addAlgorithm(ewe); - - Algorithm ensemble = new Algorithm(); - ensemble.setName("ewe"); - ensemble.setCategory("a"); - ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http:***REMOVED***data.d4science.org/eDd5b2ovMmpSVEZaZWYvY3g2bDZNcGRFNUIxNi85UFlHbWJQNStIS0N6Yz0"); - ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); - ensemble.setSkipJava("N"); - ensemble.setDescription("test"); - - Dependency d = new Dependency(); - d.setName("libpng3"); - d.setType("os"); - ensemble.addDependency(d); - algorithms.addAlgorithm(ensemble); - - ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); - - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/ISClientTest.java b/src/test/java/ISClientTest.java deleted file mode 100644 index 586b9aa..0000000 --- a/src/test/java/ISClientTest.java +++ /dev/null @@ -1,17 +0,0 @@ - - -import java.net.ProxySelector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class ISClientTest ***REMOVED*** - - public static void main(String[] args) ***REMOVED*** - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(new ISClient().listDataminersInVRE()); - ***REMOVED*** - -***REMOVED***