diff --git a/.classpath b/.classpath deleted file mode 100644 index 70eaec6..0000000 --- a/.classpath +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.project b/.project deleted file mode 100644 index 367d621..0000000 --- a/.project +++ /dev/null @@ -1,42 +0,0 @@ - - - dataminer-pool-manager - - - - - - org.eclipse.wst.jsdt.core.javascriptValidator - - - - - org.eclipse.wst.common.project.facet.core.builder - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.wst.validation.validationbuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jem.workbench.JavaEMFNature - org.eclipse.wst.common.modulecore.ModuleCoreNature - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - org.eclipse.wst.common.project.facet.core.nature - org.eclipse.wst.jsdt.core.jsNature - - diff --git a/distro/LICENSE b/distro/LICENSE deleted file mode 100644 index bc7d7b7..0000000 --- a/distro/LICENSE +++ /dev/null @@ -1 +0,0 @@ -${gcube.license***REMOVED*** \ No newline at end of file diff --git a/distro/README b/distro/README deleted file mode 100644 index 70c9e41..0000000 --- a/distro/README +++ /dev/null @@ -1,60 +0,0 @@ -The gCube System - ${name***REMOVED*** --------------------------------------------------- - -${description***REMOVED*** - -${gcube.description***REMOVED*** - -${gcube.funding***REMOVED*** - - -Version --------------------------------------------------- -${version***REMOVED*** (${buildDate***REMOVED***) - -Please see the file named "changelog.xml" in this directory for the release notes. - - -Authors --------------------------------------------------- -* Paolo Fabriani (paolo.fabriani-AT-eng.it) - Engineering Ingegneria Informatica S.p.A., Italy - - -Maintainers --------------------------------------------------- -* Paolo Fabriani (paolo.fabriani-AT-eng.it) - Engineering Ingegneria Informatica S.p.A., Italy - - -Download information --------------------------------------------------- -Source code is available from SVN: - ${scm.url***REMOVED*** - -Binaries can be downloaded from the gCube website: - ${gcube.website***REMOVED*** - - -Installation --------------------------------------------------- -Installation documentation is available on-line in the gCube Wiki: - ${gcube.wikiRoot***REMOVED***/[admin guide page name here] - - -Documentation --------------------------------------------------- -Documentation is available on-line in the gCube Wiki: - ${gcube.wikiRoot***REMOVED***[user guide page name here] - ${gcube.wikiRoot***REMOVED***[developer guide page name here] - - -Support --------------------------------------------------- -Bugs and support requests can be reported in the gCube issue tracking tool: - ${gcube.issueTracking***REMOVED*** - - -Licensing --------------------------------------------------- -This software is licensed under the terms you may find in the file named "LICENSE" in this directory. \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml deleted file mode 100644 index 1bf2d11..0000000 --- a/distro/descriptor.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - servicearchive - - - tar.gz - - - / - - - - ${distroDirectory***REMOVED*** - / - true - - README - LICENSE - profile.xml - - 755 - true - - - - - - target/${build.finalName***REMOVED***.war - /${artifactId***REMOVED*** - - - - \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml deleted file mode 100644 index 409a1c5..0000000 --- a/distro/profile.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - Library - - ${description***REMOVED*** - DataminerPoolManager - ${artifactId***REMOVED*** - 1.0.0 - - - ${artifactId***REMOVED*** - ${version***REMOVED*** - - ${groupId***REMOVED*** - ${artifactId***REMOVED*** - ${version***REMOVED*** - - - ${build.finalName***REMOVED***.jar - - - - - \ No newline at end of file diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 47b739d..0000000 --- a/pom.xml +++ /dev/null @@ -1,125 +0,0 @@ - - 4.0.0 - - - maven-parent - org.gcube.tools - 1.0.0 - - - - org.gcube.dataanalysis - dataminer-pool-manager - 1.0.0-SNAPSHOT - war - - dataminer-pool-manager - - - - - distro - UTF-8 - - - - - commons-io - commons-io - [2.5.0,2.6.0) - - - org.antlr - stringtemplate - [4.0.0, 4.1.0) - - - commons-configuration - commons-configuration - 1.10 - - - org.gcube.resources.discovery - ic-client - provided - - - org.yaml - snakeyaml - 1.16 - - - org.glassfish.jersey.media - jersey-media-json-jackson - 2.23.1 - - - org.glassfish.jersey.core - jersey-client - 2.22.1 - - - - org.glassfish.jersey.containers - jersey-container-servlet - 2.22.1 - - - - org.slf4j - slf4j-api - provided - - - - - - - - - - org.gcube.distribution - maven-smartgears-bom - LATEST - pom - import - - - - - - - - - maven-compiler-plugin - - 1.7 - 1.7 - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.2 - - - ${distroDirectory***REMOVED***/descriptor.xml - - - - - servicearchive - install - - single - - - - - - - - - diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java deleted file mode 100644 index 8be9d4c..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ /dev/null @@ -1,135 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible; - -***REMOVED*** -import java.io.File; -***REMOVED*** -import java.io.InputStream; -***REMOVED*** -import java.io.PrintStream; -import java.util.Scanner; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - -/** - * This class is responsible for the interface with ansible, retrieving log, - * etc. etc. It's not supposed to access templates and static stuff files. It - * does not know the service datamodel. - * - * @author paolo - * - */ -public class AnsibleWorker { - - /** - * The name of the inventory - */ - private static String INVENTORY_NAME = "inventory.yaml"; - - /** - * The directory containing roles - */ - private static String ROLES_DIR = "roles"; - - /** - * The name of the playbook - */ - private static String PLAYBOOK_NAME = "playbook.yaml"; - - /** - * The root of the worker. This corresponds to a standard ansible working dir. - */ - private File workerRoot; - - public AnsibleWorker(File root) { - this.workerRoot = root; - this.ensureWorkStructure(); - ***REMOVED*** - - public File getWorkdir() { - return this.workerRoot; - ***REMOVED*** - - public File getRolesDir() { - return new File(this.getWorkdir(), ROLES_DIR); - ***REMOVED*** - - public String getWorkerId() { - return this.workerRoot.getName(); - ***REMOVED*** - - public void ensureWorkStructure() { - ***REMOVED*** generate root - this.getWorkdir().mkdirs(); - ***REMOVED*** - - public void removeWorkStructure() { - ***REMOVED*** remove the working dir - this.getWorkdir().delete(); - ***REMOVED*** - - public File getPlaybookFile() { - return new File(this.getWorkdir(), PLAYBOOK_NAME); - ***REMOVED*** - - public File getInventoryFile() { - return new File(this.getWorkdir(), INVENTORY_NAME); - ***REMOVED*** - - - public void setInventory(Inventory inventory) throws IOException { - ***REMOVED*** serialize the string to the 'inventory' file - AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile()); - ***REMOVED*** - - public void setPlaybook(Playbook playbook) throws IOException { - ***REMOVED*** serialize the string to the 'playbook' file - AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile()); - ***REMOVED*** - - public void addRole(Role r) throws IOException { - ***REMOVED*** Serialize role in the workdir - AnsibleSerializeHelper.serializeRole(r, this.getRolesDir()); - ***REMOVED*** - - - - public void apply() throws IOException { - ***REMOVED*** TODO execute the playbook and return output - System.out.println(this.getWorkdir()); - try { - Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath()); - - inheritIO(p.getInputStream(), System.out); - inheritIO(p.getErrorStream(), System.err); - - ***REMOVED*** catch (IOException e) { - e.printStackTrace(); - ***REMOVED*** - ***REMOVED***System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName()); - ***REMOVED*** - - - - private static void inheritIO(final InputStream src, final PrintStream dest) { - new Thread(new Runnable() { - public void run() { - Scanner sc = new Scanner(src); - while (sc.hasNextLine()) { - dest.println(sc.nextLine()); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); -***REMOVED*** - - /** - * Destroy the worker: - * - remove the working dir - */ - public void destroy() { - this.removeWorkStructure(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java deleted file mode 100644 index 5b318f3..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -public class AnsibleHost { - - private String name; - - public AnsibleHost(String name) { - this.name = name; - ***REMOVED*** - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java deleted file mode 100644 index c1b5852..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class HostGroup { - - private String name; - - private Collection hosts; - - public HostGroup(String name) { - this.name = name; - this.hosts = new Vector<>(); - ***REMOVED*** - - public void addHost(AnsibleHost h) { - this.hosts.add(h); - ***REMOVED*** - - public String getName() { - return this.name; - ***REMOVED*** - - public Collection getHosts() { - return new Vector<>(this.hosts); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java deleted file mode 100644 index 351bc22..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class Inventory { - - private Collection groups; - - public Inventory() { - this.groups = new Vector<>(); - ***REMOVED*** - - public void addGroup(HostGroup group) { - this.groups.add(group); - ***REMOVED*** - - public void addHost(AnsibleHost h, String groupName) { - this.getGroup(groupName).addHost(h); - ***REMOVED*** - - private HostGroup getGroup(String groupName) { - for (HostGroup hg : this.groups) { - if (groupName.equals(hg.getName())) { - return hg; - ***REMOVED*** - ***REMOVED*** - HostGroup hg = new HostGroup(groupName); - this.groups.add(hg); - return hg; - ***REMOVED*** - - public Collection getHostGroups() { - return new Vector<>(this.groups); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java deleted file mode 100644 index d25ce80..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -***REMOVED*** -import java.util.Vector; - -public class Playbook { - - private String hostGroupName; - - private List roles; - - private String remote_user; - - public Playbook() { - this.roles = new Vector<>(); - ***REMOVED*** - - public void addRole(String role) { - roles.add(role); - ***REMOVED*** - - public void applyTo(String hostGroupName) { - this.hostGroupName = hostGroupName; - ***REMOVED*** - - public String getHostGroupName() { - return hostGroupName; - ***REMOVED*** - - public List getRoles() { - return new Vector<>(roles); - ***REMOVED*** - -public String getRemote_user() { - return remote_user; -***REMOVED*** - -public void setRemote_user(String remote_user) { - this.remote_user = remote_user; -***REMOVED*** - -public void setHostGroupName(String hostGroupName) { - this.hostGroupName = hostGroupName; -***REMOVED*** - -public void setRoles(List roles) { - this.roles = roles; -***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java deleted file mode 100644 index 1a4c899..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class Role { - - /** - * The name of the role - */ - private String name; - - private Collection tasks; - - private Collection meta; - - public Role() { - this.tasks = new Vector<>(); - this.meta = new Vector<>(); - ***REMOVED*** - - public Role(String name) { - this(); - this.name = name; - ***REMOVED*** - - public void addTaskFile(RoleFile tf) { - this.tasks.add(tf); - ***REMOVED*** - - public void addMeta(RoleFile tf) { - this.meta.add(tf); - ***REMOVED*** - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - - public Collection getTaskFiles() { - return new Vector<>(this.tasks); - ***REMOVED*** - - public Collection getMeta() { - return new Vector<>(this.meta); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java deleted file mode 100644 index 963a46d..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -public class RoleFile { - - /** - * The path to the file, starting from the role root - */ - private String path; - - /** - * The name of the task file - */ - private String name; - - /** - * The content of the task file - * @return - */ - private String content; - - public RoleFile() { - ***REMOVED*** - - public RoleFile(String name, String content) { - this(); - this.setName(name); - this.setContent(content); - ***REMOVED*** - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - - public String getContent() { - return content; - ***REMOVED*** - - public void setContent(String content) { - this.content = content; - ***REMOVED*** - - public String getPath() { - return path; - ***REMOVED*** - - public void setPath(String path) { - this.path = path; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java deleted file mode 100644 index af5d0cb..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ /dev/null @@ -1,276 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; - -import java.io.File; -import java.io.FileOutputStream; -***REMOVED*** -import java.io.PrintStream; -import java.util.Collection; -***REMOVED*** -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.UUID; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.AlgorithmPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CranDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomRoleManager; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.OSDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.StaticRoleManager; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; -***REMOVED*** - -public class AnsibleBridge { - private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); - - - /** - * The workdir for this service - */ - private String dpmRoot; - - public AnsibleBridge() { - this(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); - ***REMOVED*** - - public AnsibleBridge(String root) { - this.dpmRoot = root; - this.ensureServiceRoot(); - ***REMOVED*** - - private void ensureServiceRoot() { - ***REMOVED*** generate root - new File(dpmRoot).mkdirs(); - ***REMOVED*** 'template' is for template roles - this.getTemplatesDir().mkdirs(); - ***REMOVED*** 'static' is for custom roles - this.getCustomDir().mkdirs(); - ***REMOVED*** 'work' is for temporary working directories - this.getWorkDir().mkdirs(); - ***REMOVED*** - - private File getWorkDir() { - return new File(this.dpmRoot, "work"); - ***REMOVED*** - - private File getTemplatesDir() { - return new File(this.dpmRoot, "templates"); - ***REMOVED*** - - private File getCustomDir() { - return new File(this.dpmRoot, "custom"); - ***REMOVED*** - - public AnsibleWorker createWorker() { - File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); - AnsibleWorker worker = new AnsibleWorker(workerRoot); - return worker; - ***REMOVED*** - - /** - * Groups hosts by domain and algorithm sets - * @param clusters - */ - public void printInventoryByDomainAndSets(Collection clusters) { - Map> inventory = new TreeMap<>(); - for(Cluster cluster:clusters) { - for(AlgorithmSet as:cluster.getAlgorithmSets()) { - String asName = as.getName(); - for(Host h:cluster.getHosts()) { - String domain = h.getDomain().getName(); - String key = String.format("[%s@%s]", asName, domain); - Set hosts = inventory.get(key); - if(hosts==null) { - hosts = new TreeSet<>(new HostComparator()); - inventory.put(key, hosts); - ***REMOVED*** - hosts.add(h); - ***REMOVED*** - - ***REMOVED*** - ***REMOVED*** - for(String key:inventory.keySet()) { - System.out.println(key); - Collection hosts = inventory.get(key); - for(Host h:hosts) { - System.out.println(h.getName()+"."+h.getDomain().getName()); - ***REMOVED*** - System.out.println(); - ***REMOVED*** - ***REMOVED*** - - /** - * Groups hosts by algorithm sets only - * @param clusters - */ - public void printInventoryBySets(Collection clusters) { - Map> inventory = new TreeMap<>(); - for (Cluster cluster : clusters) { - for (AlgorithmSet as : cluster.getAlgorithmSets()) { - String asName = as.getName(); - for (Host h : cluster.getHosts()) { - String key = String.format("[%s]", asName); - Set hosts = inventory.get(key); - if (hosts == null) { - hosts = new TreeSet<>(new HostComparator()); - inventory.put(key, hosts); - ***REMOVED*** - hosts.add(h); - ***REMOVED*** - - ***REMOVED*** - ***REMOVED*** - for (String key : inventory.keySet()) { - System.out.println(key); - Collection hosts = inventory.get(key); - for (Host h : hosts) { - System.out.println(h.getName()+"."+h.getDomain().getName()); - ***REMOVED*** - System.out.println(); - ***REMOVED*** - ***REMOVED*** - - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException { - - - return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString()); - ***REMOVED*** - - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid) throws IOException { - AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); - - - List algoRoles = new Vector<>(); - - ***REMOVED*** add algorithms and dependencies to the worker - for (Algorithm a : as.getAlgorithms()) { - for (Role r : this.generateRoles(a)) { - algoRoles.add(r); - worker.addRole(r); - ***REMOVED*** - for (Dependency d : a.getDependencies()) { - for (Role r : this.generateRoles(d)) { - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - - ***REMOVED*** add static roles - for(Role r:this.getStaticRoleManager().getStaticRoles()) { - worker.addRole(r); - ***REMOVED*** - - ***REMOVED*** generate the inventory - Inventory inventory = new Inventory(); - for (Host h : cluster.getHosts()) { - AnsibleHost ah = new AnsibleHost(h.getName()); - inventory.addHost(ah, "universe"); - inventory.addHost(ah, "d4science"); - ***REMOVED*** - worker.setInventory(inventory); - - ***REMOVED*** generate the playbook - Playbook playbook = new Playbook(); - playbook.setRemote_user("root"); - playbook.applyTo("universe"); - for(Role r:algoRoles) { - ***REMOVED*** add only 'add' roles - if(!r.getName().endsWith("remove")) { - playbook.addRole(r.getName()); - ***REMOVED*** - ***REMOVED*** - - worker.setPlaybook(playbook); - - ***REMOVED*** execute and save log locally - PrintStream console = System.out; - File path = new File(worker.getWorkdir() + File.separator + "logs"); - path.mkdirs(); - File n = new File(path + File.separator + worker.getWorkerId()); - FileOutputStream fos = new FileOutputStream(n); - PrintStream ps = new PrintStream(fos); - System.setOut(ps); - System.setErr(ps); - ***REMOVED***System.setErr(console); - - worker.apply(); - ***REMOVED***System.setOut(console); - ***REMOVED***worker.apply(); - System.out.println("Log stored to to " + n.getAbsolutePath()); - - ***REMOVED*** destroy the worker - worker.destroy(); - return worker; - ***REMOVED*** - - - - private TemplateManager getTemplateManager() { - return new TemplateManager(this.dpmRoot+"/templates"); - ***REMOVED*** - - private CustomRoleManager getCustomRoleManager() { - return new CustomRoleManager(this.dpmRoot+"/custom"); - ***REMOVED*** - - private StaticRoleManager getStaticRoleManager() { - return new StaticRoleManager(this.dpmRoot+"/static"); - ***REMOVED*** - - /** - * Generate all roles for this dependency - * @param d - */ - public Collection generateRoles(Dependency d) { - Collection roles = new Vector<>(); - - - if("os".equalsIgnoreCase(d.getType())) { - OSDependencyPackage pkg = new OSDependencyPackage(d); - if(pkg!=null) { - roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** - - ***REMOVED*** else if("custom".equalsIgnoreCase(d.getType())) { - CustomDependencyPackage pkg = new CustomDependencyPackage(d); - if(pkg!=null) { - roles.addAll(pkg.getRoles(this.getCustomRoleManager())); - ***REMOVED*** - ***REMOVED*** - - else if("github".equalsIgnoreCase(d.getType())) { - CranDependencyPackage pkg = new CranDependencyPackage(d); - if(pkg!=null) { - roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** - ***REMOVED*** - else if("cran".equalsIgnoreCase(d.getType())) { - CranDependencyPackage pkg = new CranDependencyPackage(d); - if(pkg!=null) { - roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** - - - ***REMOVED*** - return roles; - ***REMOVED*** - - public Collection generateRoles(Algorithm a) { - AlgorithmPackage pkg = new AlgorithmPackage(a); - return pkg.getRoles(this.getTemplateManager()); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java deleted file mode 100644 index 48d9197..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -***REMOVED*** -import java.io.PrintWriter; - -import org.apache.commons.io.IOUtils; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.HostGroup; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleSerializeHelper { - - public static void serialize(Inventory inventory, File inventoryFile) throws IOException { - String out = ""; - for(HostGroup hg:inventory.getHostGroups()) { - out+=String.format("[%s]\n", hg.getName()); - for(AnsibleHost h:hg.getHosts()) { - out+=h.getName()+"\n"; - ***REMOVED*** - out+="\n"; - ***REMOVED*** - out = out.trim(); - serialize(out, inventoryFile); - ***REMOVED*** - - public static void serialize(Playbook playbook, File playbookFile) throws IOException { - String out = "- hosts: " + playbook.getHostGroupName() + "\n"; - out += " remote_user: "+playbook.getRemote_user()+"\n"; - out+=" roles:\n"; - for(String r:playbook.getRoles()) { - out+=" - " + r+"\n"; - ***REMOVED*** - out+=" vars:\n"; - out+=" os_package_state: present\n"; - out = out.trim(); - serialize(out, playbookFile); - ***REMOVED*** - - public static void serializeRole(Role r, File dir) throws IOException { - ***REMOVED*** create root - File root = new File(dir, r.getName()); - root.mkdirs(); - - ***REMOVED*** create tasks - if(r.getTaskFiles().size()>0) { - File tasks = new File(root, "tasks"); - tasks.mkdirs(); - for(RoleFile tf: r.getTaskFiles()) { - serializeTask(tf, tasks); - ***REMOVED*** - ***REMOVED*** - - ***REMOVED*** create meta - if(r.getMeta().size()>0) { - File meta = new File(root, "meta"); - meta.mkdirs(); - for(RoleFile tf: r.getMeta()) { - serializeTask(tf, meta); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - - public static void serializeTask(RoleFile tf, File dir) throws IOException { - File f = new File(dir, tf.getName()); - serialize(tf.getContent().trim(), f); - ***REMOVED*** - - public static void serialize(String s, File f) throws IOException { - PrintWriter out = new PrintWriter(f); - out.println(s); - out.close(); - ***REMOVED*** - - public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException { - Role out = new Role(); - out.setName(roleDir.getName()); - - if(!roleDir.exists()) { - throw new FileNotFoundException(); - ***REMOVED*** - - try { - File tasksDir = new File(roleDir, "tasks"); - if(tasksDir.exists()) { - for(File main:tasksDir.listFiles()) { - String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); - RoleFile tf = new RoleFile(main.getName(), content); - tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); - out.addTaskFile(tf); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch(FileNotFoundException e) { - e.printStackTrace(); - ***REMOVED*** - - try { - File metaDir = new File(roleDir, "meta"); - if(metaDir.exists()) { - for(File main:metaDir.listFiles()) { - String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); - RoleFile tf = new RoleFile(main.getName(), content); - tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); - out.addMeta(tf); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch(FileNotFoundException e) { - e.printStackTrace(); - ***REMOVED*** - - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java deleted file mode 100644 index a81bfc3..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Vector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class AlgorithmPackage { - - private Algorithm algorithm; - - public AlgorithmPackage(Algorithm a) { - this.algorithm = a; - ***REMOVED*** - - protected Map getDictionary(Algorithm a) { - Map out = new HashMap(); - out.put("name", a.getName()); - out.put("category", a.getCategory()); - out.put("class", a.getClazz()); - out.put("atype", a.getAlgorithmType()); - out.put("skipjava", a.getSkipJava()); - out.put("vre", ScopeProvider.instance.get()); - ***REMOVED***out.put("vre", "FAKE_VRE"); - out.put("packageurl", a.getPackageURL()); - out.put("description", a.getDescription()); - String deps = ""; - - for(Dependency d:a.getDependencies()) { - deps+=String.format("- { role: %s ***REMOVED***\n", d.getType()+"-"+d.getName().replaceAll("/", "-")); - ***REMOVED*** - deps = deps.trim(); - out.put("dependencies", deps); - return out; - ***REMOVED*** - - protected Algorithm getAlgorithm() { - return this.algorithm; - ***REMOVED*** - - public Collection getRoles(TemplateManager tm) { - Collection out = new Vector<>(); - for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" - String roleName = "gcube-algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode); - try { - ***REMOVED*** find template - Role template = tm.getRoleTemplate("gcube-algorithm-" + mode); - ***REMOVED*** - if(template!=null) { - Map dictionary = this.getDictionary(this.getAlgorithm()); - Role r = tm.fillRoleTemplate(template, dictionary); - r.setName(roleName); - out.add(r); - ***REMOVED*** else { - System.out.println("WARNING: template is null"); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) { -***REMOVED*** e.printStackTrace(); - System.out.println("WARNING: no template found for " + roleName); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java deleted file mode 100644 index 48e4c64..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class CranDependencyPackage extends DependencyPackage { - - public CranDependencyPackage(Dependency d) { - super(d); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java deleted file mode 100644 index 466c688..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.NoSuchElementException; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class CustomDependencyPackage extends DependencyPackage { - - public CustomDependencyPackage(Dependency d) { - super(d); - ***REMOVED*** - - private String getCustomRepositoryLocation(String ansibleRoot) { - return ansibleRoot+"/custom"; - ***REMOVED*** - - /* - public void serializeTo(String ansibleRoot) { - for(String mode:new String[]{"add", "remove", "update"***REMOVED***) { - ***REMOVED*** look for roles in the 'custom' repository - try { - ***REMOVED*** role name - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - ***REMOVED*** look for the custom role - File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName); - System.out.println("** CUSTOM ** " + src); - if(src.exists()) { - ***REMOVED*** do copy - System.out.println("copying CUSTOM role"); - File dest = new File(ansibleRoot+"/work/"+roleName); - FileUtils.copyDirectory(src, dest); - ***REMOVED*** - ***REMOVED*** catch(IOException e) { - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - */ - - public Collection getRoles(CustomRoleManager crm) { - Collection out = new Vector<>(); -***REMOVED*** for(String mode:new String[]{"add", "remove", "update"***REMOVED***) { - for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" - ***REMOVED*** role name - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - try { - ***REMOVED*** look for custom role - Role role = crm.getRole(roleName); - if(role!=null) { - out.add(role); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) { -***REMOVED*** e.printStackTrace(); - System.out.println("WARNING: no custom role found for " + roleName); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java deleted file mode 100644 index 8f583c0..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.io.File; -***REMOVED*** -import java.util.NoSuchElementException; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - -public class CustomRoleManager { - - private String root; - - public CustomRoleManager(String root) { - this.root = root; - ***REMOVED*** - - public String getRoot() { - return this.root; - ***REMOVED*** - - public Role getRole(String roleName) throws NoSuchElementException { - File f = new File(this.getRoot(), roleName); - try { - return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - ***REMOVED*** catch (IOException e) { -***REMOVED*** e.printStackTrace(); - throw new NoSuchElementException("unable to find " + roleName); - ***REMOVED*** - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java deleted file mode 100644 index a4445cc..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class DependencyPackage { - - private Dependency dependency; - - public DependencyPackage(Dependency d) { - this.dependency = d; - ***REMOVED*** - - protected Map getDictionary(Dependency d) { - Map out = new HashMap(); - out.put("name", d.getName()); - out.put("type", d.getType()); - return out; - ***REMOVED*** - - protected Dependency getDependency() { - return this.dependency; - ***REMOVED*** - - public Collection getRoles(TemplateManager tm) { - Collection out = new Vector<>(); - for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName().replaceAll("/", "-")+("add".equals(mode) ? "" : "-"+mode); - try { - ***REMOVED*** find template - Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode); - ***REMOVED*** - if(template!=null) { - Map dictionary = this.getDictionary(this.getDependency()); - Role r = tm.fillRoleTemplate(template, dictionary); - r.setName(roleName); - out.add(r); - ***REMOVED*** else { - System.out.println("WARNING: template is null"); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) { -***REMOVED*** e.printStackTrace(); - System.out.println("WARNING: no template found for " + roleName); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java deleted file mode 100644 index 6c93efd..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class OSDependencyPackage extends DependencyPackage { - - public OSDependencyPackage(Dependency d) { - super(d); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java deleted file mode 100644 index 2dc24c1..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - - import java.io.File; -***REMOVED*** -import java.util.Collection; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - - public class StaticRoleManager { - - private String root; - - public StaticRoleManager(String root) { - this.root = root; - ***REMOVED*** - - public String getRoot() { - return this.root; - ***REMOVED*** - - public Collection getStaticRoles() { - Collection out = new Vector<>(); - for(File f: new File(this.getRoot()).listFiles()) { - try { - out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f)); - ***REMOVED*** catch(IOException e) { - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - - - ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java deleted file mode 100644 index 997b42a..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.io.File; -***REMOVED*** -import java.util.Map; -import java.util.NoSuchElementException; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; -import org.stringtemplate.v4.ST; - -public class TemplateManager { - - private String root; - - public TemplateManager(String root) { - this.root = root; - ***REMOVED*** - - public String getTemplateRoot() { - return this.root; - ***REMOVED*** - - /** - * Read the given template - * @param templateName - * @return - * @throws IOException - */ -***REMOVED*** private String readTemplate(String templateName) throws IOException { -***REMOVED*** File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); -***REMOVED*** System.out.println("looking for file " + templateFile.getName()); -***REMOVED*** String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8"); -***REMOVED*** return out; -***REMOVED*** ***REMOVED*** - - /** - * Return the content of the given template - * @param templateName - * @return - * @throws NoSuchElementException if no such template exists - */ -***REMOVED*** public String getTemplate(String templateName) throws NoSuchElementException { -***REMOVED*** String template = null; -***REMOVED*** try { -***REMOVED*** template = this.readTemplate(templateName); -***REMOVED*** ***REMOVED*** catch (IOException e) { -***REMOVED*** throw new NoSuchElementException(); -***REMOVED*** ***REMOVED*** -***REMOVED*** return template; -***REMOVED*** ***REMOVED*** - - public Role fillRoleTemplate(Role template, Map dictionary) { - Role out = new Role(); - out.setName(template.getName()); - for(RoleFile tf:template.getTaskFiles()) { - out.addTaskFile(this.fillTaskTemplate(tf, dictionary)); - ***REMOVED*** - for(RoleFile tf:template.getMeta()) { - out.addMeta(this.fillTaskTemplate(tf, dictionary)); - ***REMOVED*** - return out; - ***REMOVED*** - - private RoleFile fillTaskTemplate(RoleFile template, Map dictionary) { - RoleFile out = new RoleFile(); - out.setName(template.getName()); - out.setContent(this.fillTemplate(template.getContent(), dictionary)); - return out; - ***REMOVED*** - - private String fillTemplate(String template, Map dictionary) { - if (template != null) { - ST t = new ST(template); - for (String key : dictionary.keySet()) { - t.add(key, dictionary.get(key)); - ***REMOVED*** - String output = t.render(); - return output; - ***REMOVED*** - return template; - ***REMOVED*** - - public Role getRoleTemplate(String roleName) throws NoSuchElementException { - File f = new File(this.getTemplateRoot(), roleName); - try { - return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - ***REMOVED*** catch (IOException e) { -***REMOVED*** e.printStackTrace(); - throw new NoSuchElementException("unable to find " + roleName); - ***REMOVED*** - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java deleted file mode 100644 index 3a48717..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ /dev/null @@ -1,66 +0,0 @@ -***REMOVED*** - -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - -import java.util.Collection; -***REMOVED*** -import java.util.Vector; - -import org.gcube.common.resources.gcore.ServiceEndpoint; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain; -***REMOVED*** -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; - -public class ISClient { - - /** - * Return the list of hosts (dataminers) in a given VRE - * - * @param vreName - * @return - */ - public Collection listDataminersInVRE() { - - boolean remote = false; - - if (!remote) { - Collection out = new Vector<>(); - Host h = new Host(); - ***REMOVED***h.setName("bb-dataminer.res.eng.it"); - ***REMOVED***h.setName("vm101.ui.savba.sk"); - h.setName("dataminer1-devnext.d4science.org"); - out.add(h); - return out; - ***REMOVED*** else { - - SimpleQuery query = queryFor(ServiceEndpoint.class); - - ***REMOVED***old version - ***REMOVED***query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'") - ***REMOVED***.addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); - - query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'"); - - DiscoveryClient client = clientFor(ServiceEndpoint.class); - - List resources = client.submit(query); - - Collection out = new Vector<>(); - for (ServiceEndpoint r : resources) { - Host h = new Host(); - h.setName(r.profile().runtime().hostedOn()); - out.add(h); - ***REMOVED*** - return out; - ***REMOVED*** - ***REMOVED*** - - public static void main(String[] args) { - ISClient a = new ISClient(); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(a.listDataminersInVRE()); -***REMOVED*** -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java deleted file mode 100644 index 5c69bdf..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Action { - - private String name; - - private String description; - - private String script; - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - - public String getDescription() { - return description; - ***REMOVED*** - - public void setDescription(String description) { - this.description = description; - ***REMOVED*** - - public String getScript() { - return script; - ***REMOVED*** - - public void setScript(String script) { - this.script = script; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java deleted file mode 100644 index 3d85ba7..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class Algorithm { - - private String name; - - private String description; - - private String category; - - private String clazz; - - private String algorithmType; - - private String skipJava; - - private String packageURL; - - private Collection actions; - - private Collection dependencies; - - public Algorithm() { - this.actions = new Vector<>(); - this.dependencies = new Vector<>(); - Dependency p = new Dependency(); - ***REMOVED*** - - public void addDependency(Dependency dep) { - this.dependencies.add(dep); - ***REMOVED*** - - public void addAction(Action action) { - this.actions.add(action); - ***REMOVED*** - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - - public String getDescription() { - return description; - ***REMOVED*** - - public void setDescription(String description) { - this.description = description; - ***REMOVED*** - - public String getCategory() { - return category; - ***REMOVED*** - - public void setCategory(String category) { - this.category = category; - ***REMOVED*** - - public Collection getActions() { - return actions; - ***REMOVED*** - - public Collection getDependencies() { - return dependencies; - ***REMOVED*** - - public void setDependencies(Collection deps) { - this.dependencies = deps; - ***REMOVED*** - - public String toString() { - String out = "Algorithm: " + this.getName()+"\n"; - out+=" Class Name: " + this.getClazz()+"\n"; - out+=" Description: " + this.getDescription()+"\n"; - out+=" Dependencies: " + this.getDependencies()+"\n"; - return out; - ***REMOVED*** - - public String getClazz() { - return clazz; - ***REMOVED*** - - public void setClazz(String clazz) { - this.clazz = clazz; - ***REMOVED*** - - public String getPackageURL() { - return packageURL; - ***REMOVED*** - - public void setPackageURL(String packageURL) { - this.packageURL = packageURL; - ***REMOVED*** - - public String getAlgorithmType() { - return algorithmType; - ***REMOVED*** - - public void setAlgorithmType(String algorithmType) { - this.algorithmType = algorithmType; - ***REMOVED*** - - public String getSkipJava() { - return skipJava; - ***REMOVED*** - - public void setSkipJava(String skipJava) { - this.skipJava = skipJava; - ***REMOVED*** - - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java deleted file mode 100644 index 4a5bf3f..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class AlgorithmSet { - - private String name; - - private Collection algorithms; - - public AlgorithmSet() { - this.algorithms = new Vector<>(); - ***REMOVED*** - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - - public Collection getAlgorithms() { - return new Vector<>(algorithms); - ***REMOVED*** - - public void addAlgorithm(Algorithm algoritm) { - this.algorithms.add(algoritm); - ***REMOVED*** - - public Boolean hasAlgorithm(Algorithm algorithm) { - for (Algorithm a : this.algorithms) { - if (a.getName().equals(algorithm.getName())) { - return true; - ***REMOVED*** - ***REMOVED*** - return false; - ***REMOVED*** - - public String toString() { - String out = "ALGOSET: " + this.name + "\n"; - for(Algorithm a:this.algorithms) { - out+=a+"\n"; - ***REMOVED*** - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java deleted file mode 100644 index 9a704d9..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class Cluster { - - /** - * The set of hosts belonging to the cluster. - */ - private Collection hosts; - - /** - * A name for this cluster. - */ - private String name; - - /** - * A description of this cluster. - */ - private String description; - - /** - * The set of algorithms deployed on this cluster (i.e. on all its hosts) - */ - private Collection algoSets; - - public Cluster() { - this.hosts = new Vector<>(); - this.algoSets = new Vector<>(); - ***REMOVED*** - - public void addAlgorithmSet(AlgorithmSet set) { - this.algoSets.add(set); - ***REMOVED*** - - public void addHost(Host host) { - this.hosts.add(host); - ***REMOVED*** - - public Collection getHosts() { - return hosts; - ***REMOVED*** - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - - public String getDescription() { - return description; - ***REMOVED*** - - public void setDescription(String description) { - this.description = description; - ***REMOVED*** - - public Collection getAlgorithmSets() { - return algoSets; - ***REMOVED*** - - public String toString() { - String out = "Cluster: "+this.name+"\n"; - for(Host h:this.getHosts()) { - out+=" "+h+"\n"; - ***REMOVED*** - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java deleted file mode 100644 index 7c083ab..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Dependency { - - private String name; - - private String type; - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - - public String getType() { - return type; - ***REMOVED*** - - public void setType(String type) { - this.type = type; - ***REMOVED*** - - public String toString() { - return this.type+":"+this.name; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java deleted file mode 100644 index b8e55ae..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Domain { - - private String name; - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java deleted file mode 100644 index 50f22fe..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Host { - - private String name; - - private Domain domain; - - public Host() { - ***REMOVED*** - - public String getFullyQualifiedName() { - if(this.domain!=null && this.domain.getName()!=null) - return this.getName()+"."+this.getDomain().getName(); - else - return this.getName(); - ***REMOVED*** - - public String getName() { - return name; - ***REMOVED*** - - public void setName(String name) { - this.name = name; - ***REMOVED*** - - public Domain getDomain() { - return domain; - ***REMOVED*** - - public void setDomain(Domain domain) { - this.domain = domain; - ***REMOVED*** - - public String toString() { - return this.name + "@" + this.domain; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java deleted file mode 100644 index 41eb913..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; - -public class AlgorithmComparator implements Comparator { - - @Override - public int compare(Algorithm a1, Algorithm a2) { - return a1.getName().compareTo(a2.getName()); - ***REMOVED*** - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java deleted file mode 100644 index a5ff1e1..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class DependencyComparator implements Comparator { - - @Override - public int compare(Dependency a1, Dependency a2) { - int out = a1.getType().compareTo(a2.getType()); - if(out!=0) - return out; - return a1.getName().compareTo(a2.getName()); - ***REMOVED*** - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java deleted file mode 100644 index 0d27a62..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -***REMOVED*** - -public class HostComparator implements Comparator { - - @Override - public int compare(Host h1, Host h2) { - int out = h1.getDomain().getName().compareTo(h2.getDomain().getName()); - if(out!=0) - return out; - return h1.getName().compareTo(h2.getName()); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java deleted file mode 100644 index 39a4f88..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java +++ /dev/null @@ -1,103 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.process; - -import java.util.StringTokenizer; - -public class AddAlgorithmCommand { - - private String command; - private String name; - private String category; - private String clazz; - private String scope; - private String algorithmType; - private String skipJava; - private String url; - private String description; - - public AddAlgorithmCommand(String cmd) { - StringTokenizer st = new StringTokenizer(cmd, " "); - if (st.hasMoreElements()) - command = st.nextToken(); - if (st.hasMoreElements()) - name = st.nextToken(); - if (st.hasMoreElements()) - category = st.nextToken(); - if (st.hasMoreElements()) - clazz = st.nextToken(); - if (st.hasMoreElements()) - scope = st.nextToken(); - if (st.hasMoreElements()) - algorithmType = st.nextToken(); - if (st.hasMoreElements()) - skipJava = st.nextToken(); - if (st.hasMoreElements()) - url = st.nextToken(); - - String d = ""; - while (st.hasMoreElements()) - d = d + st.nextToken() + " "; - this.setDescription(d); - - ***REMOVED*** - - public void setDescription(String d) { - if(d!=null) { - d = d.trim(); - if(d.startsWith("\"") && d.endsWith("\"")) { - d = d.substring(1, d.length()-1).trim(); - ***REMOVED*** - ***REMOVED*** - this.description = d; - ***REMOVED*** - - public String getCommand() { - return command; - ***REMOVED*** - - public String getName() { - return name; - ***REMOVED*** - - public String getCategory() { - return category; - ***REMOVED*** - - public String getClazz() { - return clazz; - ***REMOVED*** - - public String getVRE() { - return scope; - ***REMOVED*** - - public String getAlgorithmType() { - return algorithmType; - ***REMOVED*** - - public String getSkipjava() { - return skipJava; - ***REMOVED*** - - public String getUrl() { - return url; - ***REMOVED*** - - public String getDescription() { - return description; - ***REMOVED*** - - public String toString() { - String out = ""; - out += String.format("%-12s: %s\n", "command", command); - out += String.format("%-12s: %s\n", "algo name", name); - out += String.format("%-12s: %s\n", "category", category); - out += String.format("%-12s: %s\n", "class", clazz); - out += String.format("%-12s: %s\n", "scope", scope); - out += String.format("%-12s: %s\n", "algo type", algorithmType); - out += String.format("%-12s: %s\n", "skip java", skipJava); - out += String.format("%-12s: %s\n", "url", url); - out += String.format("%-12s: %s\n", "description", this.description); - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java deleted file mode 100644 index 984c440..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ /dev/null @@ -1,290 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.process; - -***REMOVED*** -import java.io.InputStream; -***REMOVED*** -import java.util.HashMap; -***REMOVED*** -import java.util.Map; -import java.util.Vector; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import org.apache.commons.lang.StringEscapeUtils; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class AlgorithmPackageParser { - - /** - * The name of the file containing algorithm metadata. Expected in the root - * directory of the package. - */ - private static final String METADATA_FILE_NAME = "Info.txt"; - - private static final String METADATA_ALGORITHM_NAME = "Algorithm Name"; - - private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description"; - - private static final String METADATA_CLASS_NAME = "Class Name"; - - private static final String METADATA_PACKAGES = "Packages"; - - private static final String METADATA_KEY_VALUE_SEPARATOR = ":"; - - private static final int BUFFER_SIZE = 4096; - - /** - * Given an URL to an algorithm package, create an Algorithm object with its - * metadata. Metadata are extracted from the 'info.txt' file, if any, in the - * package. - * - * @param url - * @return An Algorithm object or null if no 'info.txt' is found in the - * package. - * @throws IOException - */ - public Algorithm parsePackage(String url) throws IOException { - String packageMetadata = this.getPackageMetadata(url); - if (packageMetadata == null) { - System.out.println("WARNING: No metadata found for " + url); - return null; - ***REMOVED*** else { - Map> parsedMetadata = this.parseMetadata(packageMetadata); - Algorithm a = this.createAlgorithm(parsedMetadata); - a.setPackageURL(url); - return a; - ***REMOVED*** - ***REMOVED*** - - /** - * Extract the content of the metadata file from the package. - * - * @param url - * @return - * @throws IOException - */ - private String getPackageMetadata(String url) throws IOException { - InputStream is = new URL(url).openStream(); - ZipInputStream zipIs = new ZipInputStream(is); - ZipEntry entry = zipIs.getNextEntry(); - String out = null; - while (entry != null) { - if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) { - out = this.getEntryContent(zipIs); - break; - ***REMOVED*** - entry = zipIs.getNextEntry(); - ***REMOVED*** - is.close(); - zipIs.close(); - return out; - ***REMOVED*** - - /** - * Read the content of a zip entry and place it in a string. - * @param zipIn - * @return - * @throws IOException - */ - private String getEntryContent(ZipInputStream zipIn) throws IOException { - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = zipIn.read(buffer)) != -1) { - s.append(new String(buffer, 0, read)); - ***REMOVED*** - return s.toString(); - ***REMOVED*** - - /** - * Parse the content of the metadata file and create a key+multivalue map. - * @param metadata - * @return - */ - private Map> parseMetadata(String metadata) { - Map> out = new HashMap>(); - String[] lines = metadata.split("\n"); - - String key = null; - String value = null; - - for (String line : lines) { - ***REMOVED*** skip empty lines - if (line.trim().isEmpty()) { - continue; - ***REMOVED*** - ***REMOVED*** scan lines one by one, looking for key and values - String[] parts = line.split(METADATA_KEY_VALUE_SEPARATOR); - if (parts.length > 1) { - ***REMOVED*** key and value on the same line - key = parts[0].trim(); - value = line.substring(parts[0].length() + 1).trim(); - ***REMOVED*** else if (parts.length == 1) { - ***REMOVED*** either a key or a value - if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) { - ***REMOVED*** key - key = parts[0].trim(); - value = null; - ***REMOVED*** else { - ***REMOVED*** value - value = line.trim(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** add key+value to the map - if (key != null && value != null) { - List values = out.get(key); - if (values == null) { - values = new Vector<>(); - out.put(key, values); - ***REMOVED*** - values.add(value); - System.out.println(key + METADATA_KEY_VALUE_SEPARATOR + " " + values); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - /** - * Create an Algorithm starting from its metadata - * @param metadata - * @return - */ -***REMOVED*** private Algorithm createAlgorithm(Map> metadata) { -***REMOVED*** Algorithm out = new Algorithm(); -***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); -***REMOVED*** out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); -***REMOVED*** out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); -***REMOVED*** List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); -***REMOVED*** if (dependencies != null) { -***REMOVED*** for (String pkg : dependencies) { -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return out; -***REMOVED*** ***REMOVED*** - - - private Algorithm createAlgorithm(Map> metadata) { - Algorithm out = new Algorithm(); - out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); - out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); - out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); - ***REMOVED***List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); - - - List rdependencies = extractMultipleValues(metadata, "cran"); - if (rdependencies != null) { - for (String pkg : rdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("cran"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - - List defdependencies = extractMultipleValues(metadata, "Packages"); - if (defdependencies != null) { - for (String pkg : defdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - List osdependencies = extractMultipleValues(metadata, "os"); - if (osdependencies != null) { - for (String pkg : osdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - - - List gitdependencies = extractMultipleValues(metadata, "github"); - if (gitdependencies != null) { - for (String pkg : gitdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("github"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - - - List cdependencies = extractMultipleValues(metadata, "custom"); - if (cdependencies != null) { - for (String pkg : cdependencies) { - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("custom"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - -***REMOVED*** if (pkg.startsWith("r:")){ -***REMOVED*** ***REMOVED***String results = StringEscapeUtils.escapeJava(pkg); -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("cran"); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (pkg.startsWith("custom:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("custom"); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:")){ -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** ***REMOVED*** - - - - return out; - ***REMOVED*** - - - - - private static String extractSingleValue(Map> metadata, - String key) { - List l = metadata.get(key); - if (l != null && l.size() == 1) { - return l.get(0); - ***REMOVED*** else { - return null; - ***REMOVED*** - ***REMOVED*** - - - - - private static List extractMultipleValues( - Map> metadata, String key) { - List l = metadata.get(key); - if (l != null) { - return new Vector<>(l); - ***REMOVED*** else { - return null; - ***REMOVED*** - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java deleted file mode 100644 index cafe8f5..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.rest; - -***REMOVED*** -import java.net.MalformedURLException; -***REMOVED*** -import java.net.UnknownHostException; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; - -public interface PoolManager { - - String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException; - - Algorithm extractAlgorithm(String url) throws IOException; - - String getLogById(String logId) throws IOException; - - void getLogId(Algorithm algo, String vre); - - String getScriptFromURL(URL logId) throws IOException; - - URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java deleted file mode 100644 index 392a9b0..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ /dev/null @@ -1,144 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.rest; - -***REMOVED*** -import java.net.MalformedURLException; -import java.net.ProxySelector; -***REMOVED*** -import java.net.UnknownHostException; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; -***REMOVED*** -***REMOVED*** - - - - -@Path("/") -public class RestPoolManager implements PoolManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class); - - private PoolManager service = new DataminerPoolManager(); - - - @GET - @Path("/add") - @Produces("text/plain") - public String addAlgorithmToVRE( - @QueryParam("algorithm") String algorithm, - @QueryParam("vre") String vre, - @QueryParam("name") String name, - @QueryParam("description") String description, - @QueryParam("category") String category, - @QueryParam("algorithmType") String algorithmType, - @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException { - ***REMOVED*** TODO Auto-generated method stub - LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre); - Algorithm algo = service.extractAlgorithm(algorithm); - - if (algo.getCategory() == null){ - algo.setCategory(category); - ***REMOVED*** else category = algo.getCategory(); - - if (algo.getAlgorithmType() == null){ - algo.setAlgorithmType(algorithmType); - ***REMOVED*** else algorithmType = algo.getCategory(); - - if (algo.getSkipJava() == null){ - algo.setSkipJava(skipJava); - ***REMOVED*** else skipJava = algo.getSkipJava(); - - if (algo.getName() == null){ - algo.setCategory(name); - ***REMOVED*** else name = algo.getName(); - - if (algo.getDescription() == null){ - algo.setDescription(description);; - ***REMOVED*** else description = algo.getDescription(); - - return service.addAlgorithmToVRE(algo, vre); -***REMOVED*** - - - @GET - @Path("/log") - @Produces("text/plain") - public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException { - ***REMOVED*** TODO Auto-generated method stub - LOGGER.debug("Returning Log =" + logUrl); - return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl)); -***REMOVED*** - - - - - - @Override - public Algorithm extractAlgorithm(String url) throws IOException { - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** - - - public static void main(String[] args) throws IOException, InterruptedException { - RestPoolManager a = new RestPoolManager(); - ***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); - - - ***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - a.addAlgorithmToVRE( - "http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0", - "/gcube/devNext/NextNext", - null, null, "test", "transducerers", "N"); - - ***REMOVED***System.out.println(a.getLogById("34ac474d-b9df-4929-87e1-2a0ae26cf898")); -***REMOVED*** - - - @Override - public void getLogId(Algorithm algo, String vre) { - ***REMOVED*** TODO Auto-generated method stub - -***REMOVED*** - - - @Override - public String getScriptFromURL(URL logId) throws IOException { - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** - - - @Override - public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException { - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** - - - @Override - public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** - - - - - - - - - - -***REMOVED*** \ No newline at end of file diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java deleted file mode 100644 index 54387bb..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ /dev/null @@ -1,262 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.service; - -***REMOVED*** -import java.io.File; -***REMOVED*** -***REMOVED*** -import java.net.MalformedURLException; -***REMOVED*** -import java.net.URLConnection; -import java.net.UnknownHostException; -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -***REMOVED*** -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager; -***REMOVED*** -***REMOVED*** - -public class DataminerPoolManager implements PoolManager { - - - private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); - - -***REMOVED*** static Collection algorithms; -***REMOVED*** -***REMOVED*** static Collection sets; -***REMOVED*** -***REMOVED*** static { -***REMOVED*** algorithms = new Vector<>(); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public DataminerPoolManager() { -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected -***REMOVED*** * on the pool. -***REMOVED*** */ -***REMOVED*** public void publishAlgorithm(Algorithm algorithm) { -***REMOVED*** algorithms.add(algorithm); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Re-deploy the given algorithm wherever it's installed -***REMOVED*** * -***REMOVED*** * @param algorithm -***REMOVED*** */ -***REMOVED*** /* -***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement this ***REMOVED*** -***REMOVED*** */ -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Add the give algorithm to the given set -***REMOVED*** * -***REMOVED*** * @param algorithmId -***REMOVED*** * @param setId -***REMOVED*** */ -***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) { -***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); -***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName); -***REMOVED*** if (set != null && algorithm != null) { -***REMOVED*** set.addAlgorithm(algorithm); -***REMOVED*** this.updateClusters(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Apply the given set of algorithms to the given cluster -***REMOVED*** * -***REMOVED*** * @param setId -***REMOVED*** * @param clusterId -***REMOVED*** */ -***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) { -***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); -***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName); -***REMOVED*** if (set != null && cluster != null) { -***REMOVED*** cluster.addAlgorithmSet(set); -***REMOVED*** this.updateClusters(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) { -***REMOVED*** for (AlgorithmSet set : sets) { -***REMOVED*** if (name.equals(set.getName())) { -***REMOVED*** return set; -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** private Algorithm getAlgorithm(String name) { -***REMOVED*** for (Algorithm a : algorithms) { -***REMOVED*** if (name.equals(a.getName())) { -***REMOVED*** return a; -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** - - - - -***REMOVED*** -***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) { -***REMOVED*** new Thread() { -***REMOVED*** public void run() { -***REMOVED*** while (true) { -***REMOVED*** try { -***REMOVED*** addAlgorithmToVRE(algorithm, vre); -***REMOVED*** ***REMOVED*** catch (Exception e) { -***REMOVED*** ***REMOVED***log here -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED***.start(); -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** - -***REMOVED*** public String getLogId(){ -***REMOVED*** PrintStream console = System.out; -***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); -***REMOVED*** path.mkdirs(); -***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); -***REMOVED*** FileOutputStream fos = new FileOutputStream(n); -***REMOVED*** PrintStream ps = new PrintStream(fos); -***REMOVED*** System.setOut(ps); -***REMOVED*** worker.apply(); -***REMOVED*** System.setOut(console); -***REMOVED*** worker.apply(); -***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); -***REMOVED******REMOVED*** - - - - -***REMOVED*** public String getLogById(String id) throws IOException { -***REMOVED*** String strLine = null; -***REMOVED*** try{ -***REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id); -***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); -***REMOVED*** /* read log line by line */ -***REMOVED*** while ((strLine = br.readLine()) != null) { -***REMOVED*** /* parse strLine to obtain what you want */ -***REMOVED*** System.out.println (strLine); -***REMOVED*** ***REMOVED*** -***REMOVED*** br.close(); -***REMOVED*** ***REMOVED*** catch (Exception e) { -***REMOVED*** System.err.println("Error: " + e.getMessage()); -***REMOVED*** ***REMOVED*** -***REMOVED*** return strLine; -***REMOVED******REMOVED*** - - - public String getScriptFromURL(URL url) throws IOException { - if (url == null) { - return null; - ***REMOVED*** - URLConnection yc = url.openConnection(); - BufferedReader input = new BufferedReader(new InputStreamReader( - yc.getInputStream())); - String line; - StringBuffer buffer = new StringBuffer(); - while ((line = input.readLine()) != null) { - buffer.append(line + "\n"); - ***REMOVED*** - String bufferScript = buffer.substring(0, buffer.length()); - input.close(); - return bufferScript; - ***REMOVED*** - - - - - - /** - * Publish the given algorithm in the given VRE - * - * @param algorithmName - * @param vre - * - */ - public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException { - ***REMOVED*** create a fake algorithm set - final AlgorithmSet algoSet = new AlgorithmSet(); - algoSet.setName("fake"); - algoSet.addAlgorithm(algorithm); - final String uuid = UUID.randomUUID().toString(); - - new Thread(new Runnable() { - @Override - public void run() { - ***REMOVED*** TODO Auto-generated method stub - try { - addAlgorithmsToVRE(algoSet, vre, uuid); - ***REMOVED*** catch (IOException e) { - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** -***REMOVED***).start(); - ***REMOVED***this line will execute immediately, not waiting for your task to complete - System.out.println(uuid); - return uuid; - ***REMOVED*** - - - - public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{ - - File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs"); - path.mkdirs(); - File n = new File(path + File.separator +a); - ***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress(); - - return new File(n.getPath()).toURI().toURL(); - ***REMOVED*** - - - public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException { - - ***REMOVED*** create the cluster (dataminers in the vre) - Cluster cluster = new Cluster(); - for(Host h:new ISClient().listDataminersInVRE()) { - cluster.addHost(h); - ***REMOVED*** - - ***REMOVED*** apply the changes - AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId(); - - ***REMOVED*** - - public Algorithm extractAlgorithm(String url) throws IOException { - return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** - - -@Override -public void getLogId(Algorithm algo, String vre) { - ***REMOVED*** TODO Auto-generated method stub - -***REMOVED*** - - -@Override -public String getLogById(String logId) throws IOException { - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** - - - - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java deleted file mode 100644 index 98118a3..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.util; - -***REMOVED*** -import java.net.Authenticator; -import java.net.InetSocketAddress; -import java.net.PasswordAuthentication; -import java.net.Proxy; -import java.net.ProxySelector; -import java.net.SocketAddress; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -***REMOVED*** - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy; - -interface NetworkConfiguration { - - public String getProxyHost(); - - public String getProxyPort(); - - public String getProxyUser(); - - public String getProxyPassword(); - - public String getNonProxyHosts(); - -***REMOVED*** - -class FileBasedProxyConfiguration implements NetworkConfiguration { - - private static PropertiesConfiguration configuration; - - public FileBasedProxyConfiguration(String path) { - try { - ***REMOVED*** load the configuration - configuration = new PropertiesConfiguration(path); - ***REMOVED*** set the reloading strategy to enable hot-configuration - FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy(); - configuration.setReloadingStrategy(fcrs); - ***REMOVED*** catch (ConfigurationException e) { - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - - @Override - public String getProxyHost() { - return configuration.getString("proxyHost"); - ***REMOVED*** - - @Override - public String getProxyPort() { - return configuration.getString("proxyPort"); - ***REMOVED*** - - @Override - public String getProxyUser() { - return configuration.getString("proxyUser"); - ***REMOVED*** - - @Override - public String getProxyPassword() { - return configuration.getString("proxyPassword"); - ***REMOVED*** - - @Override - public String getNonProxyHosts() { - return configuration.getString("nonProxyHosts"); - ***REMOVED*** - -***REMOVED*** - -public class PropertiesBasedProxySelector extends ProxySelector { - - List proxies = null; - - List nonProxyHosts = null; - - public PropertiesBasedProxySelector(String proxySettingsPath) { - this(new FileBasedProxyConfiguration(proxySettingsPath)); - ***REMOVED*** - - public PropertiesBasedProxySelector(NetworkConfiguration config) { - if (config == null || config.getProxyHost() == null) { - this.proxies = null; - return; - ***REMOVED*** - - String host = config.getProxyHost(); - - int port = 80; - - if (config.getProxyPort() != null) { - port = Integer.valueOf(config.getProxyPort()); - ***REMOVED*** - - if (config.getNonProxyHosts() != null) { - this.nonProxyHosts = Arrays - .asList(config.getNonProxyHosts().split("\\|")); - ***REMOVED*** - - this.proxies = new ArrayList(); - this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host, - port))); - - if (config.getProxyUser() != null) { - final String username = config.getProxyUser(); - final String password = config.getProxyPassword(); - - Authenticator.setDefault(new Authenticator() { - @Override - protected PasswordAuthentication getPasswordAuthentication() { - return new PasswordAuthentication(username, password.toCharArray()); - ***REMOVED*** - ***REMOVED***); - - ***REMOVED*** - ***REMOVED*** - - @Override - public List select(URI uri) { - if (this.nonProxyHosts == null) { - return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** else { - for (String entry : this.nonProxyHosts) { - entry = entry.trim(); - if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) { - return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** - if (uri.getHost().equals(entry)) { - return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** - ***REMOVED*** - return this.proxies; - ***REMOVED*** - ***REMOVED*** - - @Override - public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) { - - ***REMOVED*** -***REMOVED*** diff --git a/src/main/webapp/WEB-INF/gcube-app.xml b/src/main/webapp/WEB-INF/gcube-app.xml deleted file mode 100644 index 15f231c..0000000 --- a/src/main/webapp/WEB-INF/gcube-app.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - dataminer-pool-manager - dataanalysis - 0.0.1 - - - - - - - - \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/gcube-handlers.xml b/src/main/webapp/WEB-INF/gcube-handlers.xml deleted file mode 100644 index c294252..0000000 --- a/src/main/webapp/WEB-INF/gcube-handlers.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml deleted file mode 100644 index 2a231cb..0000000 --- a/src/main/webapp/WEB-INF/web.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - Archetype Created Web Application - - - - REST-API - org.glassfish.jersey.servlet.ServletContainer - - jersey.config.server.provider.packages - org.gcube.dataanalysis.dataminer.poolmanager.rest - - 1 - - - - REST-API - /rest/* - - - diff --git a/src/test/java/AlgorithmPackageParserTest.java b/src/test/java/AlgorithmPackageParserTest.java deleted file mode 100644 index a0de0cc..0000000 --- a/src/test/java/AlgorithmPackageParserTest.java +++ /dev/null @@ -1,117 +0,0 @@ - - -***REMOVED*** -import java.io.InputStream; -***REMOVED*** -***REMOVED*** -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; - -public class AlgorithmPackageParserTest { - - private static int BUFFER_SIZE = 2048; - - public void extractAllAlgorithms() throws IOException { - String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; - List commands = this.extractAddAlgorithmCommands(url); - AlgorithmSet algorithms = new AlgorithmSet(); - for (String cmd : commands) { - System.out.println("-----------------------------------------"); - System.out.println(cmd); - AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - System.out.println(aac); - - ***REMOVED*** start creating the algo from the command - Algorithm algo = new Algorithm(); - algo.setAlgorithmType(aac.getAlgorithmType()); - algo.setCategory(aac.getCategory()); - algo.setClazz(aac.getClazz()); - algo.setDescription(aac.getDescription()); - algo.setName(aac.getName()); - algo.setPackageURL(aac.getUrl()); - algo.setSkipJava(aac.getSkipjava()); - - ***REMOVED*** then override with info from the package - if (aac.getUrl().length() > 4) { - Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); - if (packagedAlgo != null) { - algo.setDependencies(packagedAlgo.getDependencies()); - ***REMOVED*** - ***REMOVED*** - algorithms.addAlgorithm(algo); - break; - ***REMOVED*** - ***REMOVED***to uncomment - ***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - ***REMOVED*** "/gcube/devNext/NextNext"); - ***REMOVED*** - - /** - * Extract 'addAlgorithm' commands from a file containing wiki-table-style - * entries for algorithm. - * - * @return - * @throws IOException - */ - private List extractAddAlgorithmCommands(String listUrl) - throws IOException { - URL url = new URL(listUrl); - InputStream is = url.openStream(); - - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = is.read(buffer)) != -1) { - s.append(new String(buffer, 0, read)); - ***REMOVED*** - List out = new Vector<>(); - String[] lines = s.toString().split("\n"); - for (String line : lines) { - System.out.println("--------------------"); - if (!line.isEmpty()) { - String[] parts = line.split("\\|"); - int c = 1; - for (String part : parts) { - if (part == null || part.trim().isEmpty()) { - continue; - ***REMOVED*** - System.out.println(c + ". " + part); - c++; - if (part.contains("addAlgorithm.sh")) { - String cmd = part.trim(); - cmd = cmd.replaceAll("", ""); - cmd = cmd.replaceAll("", ""); - System.out.println(cmd); - ***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - ***REMOVED*** System.out.println(aac); - out.add(cmd); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - /** - * Create an Algorithm starting from the algorithm jar. - * - * @param url - * @return - * @throws IOException - */ - private Algorithm extractAlgorithm(String url) throws IOException { - return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** - - public static void main(String[] args) throws Exception { -***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector( -***REMOVED*** "/home/ngalante/.proxy-settings")); - - new AlgorithmPackageParserTest().extractAllAlgorithms(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/AnsibleWorkerTest.java b/src/test/java/AnsibleWorkerTest.java deleted file mode 100644 index 1617477..0000000 --- a/src/test/java/AnsibleWorkerTest.java +++ /dev/null @@ -1,33 +0,0 @@ - - -import java.io.File; -***REMOVED*** -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleWorkerTest { - - public static void main(String[] args) throws IOException { - AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString())); - - System.out.println("created worker named " + worker.getWorkerId()); - - worker.setInventory(new Inventory()); - worker.setPlaybook(new Playbook()); - - Role r = new Role(); - r.setName("latex"); - - RoleFile tf = new RoleFile("main", "do something special for " + r.getName()); - r.addTaskFile(tf); - worker.addRole(r); - - worker.apply(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/DataminerPoolManagerTest.java b/src/test/java/DataminerPoolManagerTest.java deleted file mode 100644 index cc0c938..0000000 --- a/src/test/java/DataminerPoolManagerTest.java +++ /dev/null @@ -1,227 +0,0 @@ - - -import java.net.ProxySelector; -import java.util.UUID; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class DataminerPoolManagerTest { - - /* - private static Map domains = new HashMap<>(); - private static Map clusters = new HashMap<>(); - - private static Dependency createDependency(String depName) { - String[] parts = depName.split(":"); - Dependency out = new Dependency(); - if(parts.length>1) { - out.setType(parts[0]); - out.setName(parts[1]); - ***REMOVED*** else { - out.setType("os"); - out.setName(depName); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Algorithm createAlgorithm(String name, String ... deps) { - Algorithm a = new Algorithm(); - a.setName(name); - for(String dep:deps) { - a.addDependency(createDependency(dep)); - ***REMOVED*** - return a; - ***REMOVED*** - - private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) { - AlgorithmSet out = new AlgorithmSet(); - out.setName(name); - for(Algorithm a:algs) { - out.addAlgorithm(a); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Domain getDomain(String name) { - if(domains.get(name)==null) { - Domain d = new Domain(); - d.setName(name); - domains.put(name, d); - return d; - ***REMOVED*** else { - return domains.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Host createHost(String hostname, String domainName) { - Host out = new Host(); - out.setName(hostname); - Domain d = getDomain(domainName); - out.setDomain(d); - return out; - ***REMOVED*** - - private static Cluster getCluster(String name) { - if(clusters.get(name)==null) { - Cluster d = new Cluster(); - d.setName(name); - clusters.put(name, d); - return d; - ***REMOVED*** else { - return clusters.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Collection extractDependencies() { - Collection out = new TreeSet<>(new DependencyComparator()); - for(Cluster c:clusters.values()) { - for(AlgorithmSet as:c.getAlgorithmSets()) { - for(Algorithm a:as.getAlgorithms()) { - for(Dependency d:a.getDependencies()) { - out.add(d); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - private static Collection extractAlgorithms() { - Collection out = new TreeSet<>(new AlgorithmComparator()); - for(Cluster c:clusters.values()) { - for(AlgorithmSet as:c.getAlgorithmSets()) { - for(Algorithm a:as.getAlgorithms()) { - out.add(a); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - static { - - Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package"); - Algorithm ensemble = createAlgorithm("ensemble", "python"); - Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah"); - - AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe); - AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble); - AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe); - - Cluster cluster1 = getCluster("cluster-1"); - cluster1.addHost(createHost("host1", "domain1")); - cluster1.addHost(createHost("host2", "domain1")); - cluster1.addHost(createHost("host3", "domain1")); - cluster1.addHost(createHost("host1", "domain2")); - cluster1.addHost(createHost("host2", "domain2")); - - Cluster cluster2 = getCluster("cluster-2"); - cluster2.addHost(createHost("host4", "domain1")); - cluster2.addHost(createHost("host5", "domain1")); - cluster2.addHost(createHost("host6", "domain1")); - cluster2.addHost(createHost("host3", "domain2")); - cluster2.addHost(createHost("host4", "domain2")); - cluster2.addHost(createHost("host5", "domain2")); - - cluster1.addAlgorithmSet(as1); - cluster1.addAlgorithmSet(as2); - - cluster2.addAlgorithmSet(as1); - cluster2.addAlgorithmSet(as3); - - ***REMOVED*** - */ - - public static void main(String[] args) throws Exception { - /* - AnsibleBridge ab = new AnsibleBridge(); - ab.printInventoryByDomainAndSets(clusters.values()); - System.out.println("-----------"); - ab.printInventoryBySets(clusters.values()); - - AnsibleWorker worker = ab.createWorker(); - - for(Algorithm a:extractAlgorithms()) { - for(Role r:ab.generateRoles(a)) { - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - - for(Dependency d:extractDependencies()) { - for(Role r:ab.generateRoles(d)) { - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - */ - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm algorithm = new Algorithm(); -***REMOVED*** algorithm.setName("ichtyop"); -***REMOVED*** algorithm.setClazz("org.gcube..."); -***REMOVED*** algorithm.setDescription("some description"); -***REMOVED*** -***REMOVED*** Dependency d = new Dependency(); -***REMOVED*** d.setName("libpng"); -***REMOVED*** d.setType("os"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-other-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm ewe = new Algorithm(); -***REMOVED*** ewe.setName("ewe"); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** ewe.addDependency(d); - - AlgorithmSet algorithms = new AlgorithmSet(); - algorithms.setName("dummy-set"); - - -***REMOVED*** algorithms.addAlgorithm(algorithm); -***REMOVED*** algorithms.addAlgorithm(ewe); - - Algorithm ensemble = new Algorithm(); - ensemble.setName("ewe"); - ensemble.setCategory("a"); - ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http:***REMOVED***data.d4science.org/eDd5b2ovMmpSVEZaZWYvY3g2bDZNcGRFNUIxNi85UFlHbWJQNStIS0N6Yz0"); - ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); - ensemble.setSkipJava("N"); - ensemble.setDescription("test"); - - Dependency d = new Dependency(); - d.setName("libpng3"); - d.setType("os"); - ensemble.addDependency(d); - algorithms.addAlgorithm(ensemble); - - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID()); - - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/ISClientTest.java b/src/test/java/ISClientTest.java deleted file mode 100644 index f1a1574..0000000 --- a/src/test/java/ISClientTest.java +++ /dev/null @@ -1,17 +0,0 @@ - - -import java.net.ProxySelector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class ISClientTest { - - public static void main(String[] args) { - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(new ISClient().listDataminersInVRE()); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java deleted file mode 100644 index ed15ed6..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -***REMOVED*** -import java.io.InputStream; -import java.net.ProxySelector; -***REMOVED*** -***REMOVED*** -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class AlgorithmPackageParserTest { - - private static int BUFFER_SIZE = 2048; - - public void extractAllAlgorithms() throws IOException { - String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; - List commands = this.extractAddAlgorithmCommands(url); - AlgorithmSet algorithms = new AlgorithmSet(); - for (String cmd : commands) { - System.out.println("-----------------------------------------"); - System.out.println(cmd); - AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - System.out.println(aac); - - ***REMOVED*** start creating the algo from the command - Algorithm algo = new Algorithm(); - algo.setAlgorithmType(aac.getAlgorithmType()); - algo.setCategory(aac.getCategory()); - algo.setClazz(aac.getClazz()); - algo.setDescription(aac.getDescription()); - algo.setName(aac.getName()); - algo.setPackageURL(aac.getUrl()); - algo.setSkipJava(aac.getSkipjava()); - - ***REMOVED*** then override with info from the package - if (aac.getUrl().length() > 4) { - Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); - if (packagedAlgo != null) { - algo.setDependencies(packagedAlgo.getDependencies()); - ***REMOVED*** - ***REMOVED*** - algorithms.addAlgorithm(algo); - break; - ***REMOVED*** - ***REMOVED***to uncomment - ***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - ***REMOVED*** "/gcube/devNext/NextNext"); - ***REMOVED*** - - /** - * Extract 'addAlgorithm' commands from a file containing wiki-table-style - * entries for algorithm. - * - * @return - * @throws IOException - */ - private List extractAddAlgorithmCommands(String listUrl) - throws IOException { - URL url = new URL(listUrl); - InputStream is = url.openStream(); - - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = is.read(buffer)) != -1) { - s.append(new String(buffer, 0, read)); - ***REMOVED*** - List out = new Vector<>(); - String[] lines = s.toString().split("\n"); - for (String line : lines) { - System.out.println("--------------------"); - if (!line.isEmpty()) { - String[] parts = line.split("\\|"); - int c = 1; - for (String part : parts) { - if (part == null || part.trim().isEmpty()) { - continue; - ***REMOVED*** - System.out.println(c + ". " + part); - c++; - if (part.contains("addAlgorithm.sh")) { - String cmd = part.trim(); - cmd = cmd.replaceAll("", ""); - cmd = cmd.replaceAll("", ""); - System.out.println(cmd); - ***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - ***REMOVED*** System.out.println(aac); - out.add(cmd); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - /** - * Create an Algorithm starting from the algorithm jar. - * - * @param url - * @return - * @throws IOException - */ - private Algorithm extractAlgorithm(String url) throws IOException { - return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** - - public static void main(String[] args) throws Exception { -***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector( -***REMOVED*** "/home/ngalante/.proxy-settings")); - - new AlgorithmPackageParserTest().extractAllAlgorithms(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java deleted file mode 100644 index 0cae1b3..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.io.File; -***REMOVED*** -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleWorkerTest { - - public static void main(String[] args) throws IOException { - AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString())); - - System.out.println("created worker named " + worker.getWorkerId()); - - worker.setInventory(new Inventory()); - worker.setPlaybook(new Playbook()); - - Role r = new Role(); - r.setName("latex"); - - RoleFile tf = new RoleFile("main", "do something special for " + r.getName()); - r.addTaskFile(tf); - worker.addRole(r); - - worker.apply(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java deleted file mode 100644 index ef41e96..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java +++ /dev/null @@ -1,227 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.net.ProxySelector; -import java.util.UUID; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class DataminerPoolManagerTest { - - /* - private static Map domains = new HashMap<>(); - private static Map clusters = new HashMap<>(); - - private static Dependency createDependency(String depName) { - String[] parts = depName.split(":"); - Dependency out = new Dependency(); - if(parts.length>1) { - out.setType(parts[0]); - out.setName(parts[1]); - ***REMOVED*** else { - out.setType("os"); - out.setName(depName); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Algorithm createAlgorithm(String name, String ... deps) { - Algorithm a = new Algorithm(); - a.setName(name); - for(String dep:deps) { - a.addDependency(createDependency(dep)); - ***REMOVED*** - return a; - ***REMOVED*** - - private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) { - AlgorithmSet out = new AlgorithmSet(); - out.setName(name); - for(Algorithm a:algs) { - out.addAlgorithm(a); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Domain getDomain(String name) { - if(domains.get(name)==null) { - Domain d = new Domain(); - d.setName(name); - domains.put(name, d); - return d; - ***REMOVED*** else { - return domains.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Host createHost(String hostname, String domainName) { - Host out = new Host(); - out.setName(hostname); - Domain d = getDomain(domainName); - out.setDomain(d); - return out; - ***REMOVED*** - - private static Cluster getCluster(String name) { - if(clusters.get(name)==null) { - Cluster d = new Cluster(); - d.setName(name); - clusters.put(name, d); - return d; - ***REMOVED*** else { - return clusters.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Collection extractDependencies() { - Collection out = new TreeSet<>(new DependencyComparator()); - for(Cluster c:clusters.values()) { - for(AlgorithmSet as:c.getAlgorithmSets()) { - for(Algorithm a:as.getAlgorithms()) { - for(Dependency d:a.getDependencies()) { - out.add(d); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - private static Collection extractAlgorithms() { - Collection out = new TreeSet<>(new AlgorithmComparator()); - for(Cluster c:clusters.values()) { - for(AlgorithmSet as:c.getAlgorithmSets()) { - for(Algorithm a:as.getAlgorithms()) { - out.add(a); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - static { - - Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package"); - Algorithm ensemble = createAlgorithm("ensemble", "python"); - Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah"); - - AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe); - AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble); - AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe); - - Cluster cluster1 = getCluster("cluster-1"); - cluster1.addHost(createHost("host1", "domain1")); - cluster1.addHost(createHost("host2", "domain1")); - cluster1.addHost(createHost("host3", "domain1")); - cluster1.addHost(createHost("host1", "domain2")); - cluster1.addHost(createHost("host2", "domain2")); - - Cluster cluster2 = getCluster("cluster-2"); - cluster2.addHost(createHost("host4", "domain1")); - cluster2.addHost(createHost("host5", "domain1")); - cluster2.addHost(createHost("host6", "domain1")); - cluster2.addHost(createHost("host3", "domain2")); - cluster2.addHost(createHost("host4", "domain2")); - cluster2.addHost(createHost("host5", "domain2")); - - cluster1.addAlgorithmSet(as1); - cluster1.addAlgorithmSet(as2); - - cluster2.addAlgorithmSet(as1); - cluster2.addAlgorithmSet(as3); - - ***REMOVED*** - */ - - public static void main(String[] args) throws Exception { - /* - AnsibleBridge ab = new AnsibleBridge(); - ab.printInventoryByDomainAndSets(clusters.values()); - System.out.println("-----------"); - ab.printInventoryBySets(clusters.values()); - - AnsibleWorker worker = ab.createWorker(); - - for(Algorithm a:extractAlgorithms()) { - for(Role r:ab.generateRoles(a)) { - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - - for(Dependency d:extractDependencies()) { - for(Role r:ab.generateRoles(d)) { - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - */ - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm algorithm = new Algorithm(); -***REMOVED*** algorithm.setName("ichtyop"); -***REMOVED*** algorithm.setClazz("org.gcube..."); -***REMOVED*** algorithm.setDescription("some description"); -***REMOVED*** -***REMOVED*** Dependency d = new Dependency(); -***REMOVED*** d.setName("libpng"); -***REMOVED*** d.setType("os"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-other-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm ewe = new Algorithm(); -***REMOVED*** ewe.setName("ewe"); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** ewe.addDependency(d); - - AlgorithmSet algorithms = new AlgorithmSet(); - algorithms.setName("dummy-set"); - - -***REMOVED*** algorithms.addAlgorithm(algorithm); -***REMOVED*** algorithms.addAlgorithm(ewe); - - Algorithm ensemble = new Algorithm(); - ensemble.setName("ensemble"); - ensemble.setCategory("ICHTHYOP_MODEL"); - ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http:***REMOVED***data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0"); - ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); - ensemble.setSkipJava("N"); - ensemble.setDescription("test"); - - Dependency d = new Dependency(); - d.setName("libpng3"); - d.setType("os"); - ensemble.addDependency(d); - algorithms.addAlgorithm(ensemble); - - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID()); - - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java deleted file mode 100644 index 426cc6c..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.net.ProxySelector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class ISClientTest { - - public static void main(String[] args) { - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(new ISClient().listDataminersInVRE()); - ***REMOVED*** - -***REMOVED***