diff --git a/.classpath b/.classpath deleted file mode 100644 index 70eaec6..0000000 --- a/.classpath +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.project b/.project deleted file mode 100644 index 367d621..0000000 --- a/.project +++ /dev/null @@ -1,42 +0,0 @@ - - - dataminer-pool-manager - - - - - - org.eclipse.wst.jsdt.core.javascriptValidator - - - - - org.eclipse.wst.common.project.facet.core.builder - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.wst.validation.validationbuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jem.workbench.JavaEMFNature - org.eclipse.wst.common.modulecore.ModuleCoreNature - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - org.eclipse.wst.common.project.facet.core.nature - org.eclipse.wst.jsdt.core.jsNature - - diff --git a/distro/LICENSE b/distro/LICENSE deleted file mode 100644 index 645394d..0000000 --- a/distro/LICENSE +++ /dev/null @@ -1 +0,0 @@ -$***REMOVED***gcube.license***REMOVED*** \ No newline at end of file diff --git a/distro/README b/distro/README deleted file mode 100644 index 5d45173..0000000 --- a/distro/README +++ /dev/null @@ -1,60 +0,0 @@ -The gCube System - $***REMOVED***name***REMOVED*** --------------------------------------------------- - -$***REMOVED***description***REMOVED*** - -$***REMOVED***gcube.description***REMOVED*** - -$***REMOVED***gcube.funding***REMOVED*** - - -Version --------------------------------------------------- -$***REMOVED***version***REMOVED*** ($***REMOVED***buildDate***REMOVED***) - -Please see the file named "changelog.xml" in this directory for the release notes. - - -Authors --------------------------------------------------- -* Paolo Fabriani (paolo.fabriani-AT-eng.it) - Engineering Ingegneria Informatica S.p.A., Italy - - -Maintainers --------------------------------------------------- -* Paolo Fabriani (paolo.fabriani-AT-eng.it) - Engineering Ingegneria Informatica S.p.A., Italy - - -Download information --------------------------------------------------- -Source code is available from SVN: - $***REMOVED***scm.url***REMOVED*** - -Binaries can be downloaded from the gCube website: - $***REMOVED***gcube.website***REMOVED*** - - -Installation --------------------------------------------------- -Installation documentation is available on-line in the gCube Wiki: - $***REMOVED***gcube.wikiRoot***REMOVED***/[admin guide page name here] - - -Documentation --------------------------------------------------- -Documentation is available on-line in the gCube Wiki: - $***REMOVED***gcube.wikiRoot***REMOVED***[user guide page name here] - $***REMOVED***gcube.wikiRoot***REMOVED***[developer guide page name here] - - -Support --------------------------------------------------- -Bugs and support requests can be reported in the gCube issue tracking tool: - $***REMOVED***gcube.issueTracking***REMOVED*** - - -Licensing --------------------------------------------------- -This software is licensed under the terms you may find in the file named "LICENSE" in this directory. \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml deleted file mode 100644 index 5c6de76..0000000 --- a/distro/descriptor.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - servicearchive - - - tar.gz - - - / - - - - $***REMOVED***distroDirectory***REMOVED*** - / - true - - README - LICENSE - profile.xml - - 755 - true - - - - - - target/$***REMOVED***build.finalName***REMOVED***.war - /$***REMOVED***artifactId***REMOVED*** - - - - \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml deleted file mode 100644 index 83b9c7e..0000000 --- a/distro/profile.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - Library - - $***REMOVED***description***REMOVED*** - DataminerPoolManager - $***REMOVED***artifactId***REMOVED*** - 1.0.0 - - - $***REMOVED***artifactId***REMOVED*** - $***REMOVED***version***REMOVED*** - - $***REMOVED***groupId***REMOVED*** - $***REMOVED***artifactId***REMOVED*** - $***REMOVED***version***REMOVED*** - - - $***REMOVED***build.finalName***REMOVED***.jar - - - - - \ No newline at end of file diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 6d58b22..0000000 --- a/pom.xml +++ /dev/null @@ -1,125 +0,0 @@ - - 4.0.0 - - - maven-parent - org.gcube.tools - 1.0.0 - - - - org.gcube.dataanalysis - dataminer-pool-manager - 1.0.0-SNAPSHOT - war - - dataminer-pool-manager - - - - - distro - UTF-8 - - - - - commons-io - commons-io - [2.5.0,2.6.0) - - - org.antlr - stringtemplate - [4.0.0, 4.1.0) - - - commons-configuration - commons-configuration - 1.10 - - - org.gcube.resources.discovery - ic-client - provided - - - org.yaml - snakeyaml - 1.16 - - - org.glassfish.jersey.media - jersey-media-json-jackson - 2.23.1 - - - org.glassfish.jersey.core - jersey-client - 2.22.1 - - - - org.glassfish.jersey.containers - jersey-container-servlet - 2.22.1 - - - - org.slf4j - slf4j-api - provided - - - - - - - - - - org.gcube.distribution - maven-smartgears-bom - LATEST - pom - import - - - - - - - - - maven-compiler-plugin - - 1.7 - 1.7 - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.2 - - - $***REMOVED***distroDirectory***REMOVED***/descriptor.xml - - - - - servicearchive - install - - single - - - - - - - - - diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java deleted file mode 100644 index cd2458a..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ /dev/null @@ -1,135 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible; - -import java.io.BufferedReader; -import java.io.File; -***REMOVED*** -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.util.Scanner; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - -/** - * This class is responsible for the interface with ansible, retrieving log, - * etc. etc. It's not supposed to access templates and static stuff files. It - * does not know the service datamodel. - * - * @author paolo - * - */ -public class AnsibleWorker ***REMOVED*** - - /** - * The name of the inventory - */ - private static String INVENTORY_NAME = "inventory.yaml"; - - /** - * The directory containing roles - */ - private static String ROLES_DIR = "roles"; - - /** - * The name of the playbook - */ - private static String PLAYBOOK_NAME = "playbook.yaml"; - - /** - * The root of the worker. This corresponds to a standard ansible working dir. - */ - private File workerRoot; - - public AnsibleWorker(File root) ***REMOVED*** - this.workerRoot = root; - this.ensureWorkStructure(); - ***REMOVED*** - - public File getWorkdir() ***REMOVED*** - return this.workerRoot; - ***REMOVED*** - - public File getRolesDir() ***REMOVED*** - return new File(this.getWorkdir(), ROLES_DIR); - ***REMOVED*** - - public String getWorkerId() ***REMOVED*** - return this.workerRoot.getName(); - ***REMOVED*** - - public void ensureWorkStructure() ***REMOVED*** - ***REMOVED*** generate root - this.getWorkdir().mkdirs(); - ***REMOVED*** - - public void removeWorkStructure() ***REMOVED*** - ***REMOVED*** remove the working dir - this.getWorkdir().delete(); - ***REMOVED*** - - public File getPlaybookFile() ***REMOVED*** - return new File(this.getWorkdir(), PLAYBOOK_NAME); - ***REMOVED*** - - public File getInventoryFile() ***REMOVED*** - return new File(this.getWorkdir(), INVENTORY_NAME); - ***REMOVED*** - - - public void setInventory(Inventory inventory) throws IOException ***REMOVED*** - ***REMOVED*** serialize the string to the 'inventory' file - AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile()); - ***REMOVED*** - - public void setPlaybook(Playbook playbook) throws IOException ***REMOVED*** - ***REMOVED*** serialize the string to the 'playbook' file - AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile()); - ***REMOVED*** - - public void addRole(Role r) throws IOException ***REMOVED*** - ***REMOVED*** Serialize role in the workdir - AnsibleSerializeHelper.serializeRole(r, this.getRolesDir()); - ***REMOVED*** - - - - public void apply() throws IOException ***REMOVED*** - ***REMOVED*** TODO execute the playbook and return output - System.out.println(this.getWorkdir()); - try ***REMOVED*** - Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath()); - - inheritIO(p.getInputStream(), System.out); - inheritIO(p.getErrorStream(), System.err); - - ***REMOVED*** catch (IOException e) ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** - ***REMOVED***System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName()); - ***REMOVED*** - - - - private static void inheritIO(final InputStream src, final PrintStream dest) ***REMOVED*** - new Thread(new Runnable() ***REMOVED*** - public void run() ***REMOVED*** - Scanner sc = new Scanner(src); - while (sc.hasNextLine()) ***REMOVED*** - dest.println(sc.nextLine()); - ***REMOVED*** - ***REMOVED*** - ***REMOVED***).start(); -***REMOVED*** - - /** - * Destroy the worker: - * - remove the working dir - */ - public void destroy() ***REMOVED*** - this.removeWorkStructure(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java deleted file mode 100644 index 9e6a8d7..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -public class AnsibleHost ***REMOVED*** - - private String name; - - public AnsibleHost(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java deleted file mode 100644 index 1b786e4..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class HostGroup ***REMOVED*** - - private String name; - - private Collection hosts; - - public HostGroup(String name) ***REMOVED*** - this.name = name; - this.hosts = new Vector<>(); - ***REMOVED*** - - public void addHost(AnsibleHost h) ***REMOVED*** - this.hosts.add(h); - ***REMOVED*** - - public String getName() ***REMOVED*** - return this.name; - ***REMOVED*** - - public Collection getHosts() ***REMOVED*** - return new Vector<>(this.hosts); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java deleted file mode 100644 index a53e151..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class Inventory ***REMOVED*** - - private Collection groups; - - public Inventory() ***REMOVED*** - this.groups = new Vector<>(); - ***REMOVED*** - - public void addGroup(HostGroup group) ***REMOVED*** - this.groups.add(group); - ***REMOVED*** - - public void addHost(AnsibleHost h, String groupName) ***REMOVED*** - this.getGroup(groupName).addHost(h); - ***REMOVED*** - - private HostGroup getGroup(String groupName) ***REMOVED*** - for (HostGroup hg : this.groups) ***REMOVED*** - if (groupName.equals(hg.getName())) ***REMOVED*** - return hg; - ***REMOVED*** - ***REMOVED*** - HostGroup hg = new HostGroup(groupName); - this.groups.add(hg); - return hg; - ***REMOVED*** - - public Collection getHostGroups() ***REMOVED*** - return new Vector<>(this.groups); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java deleted file mode 100644 index 9a7f045..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -***REMOVED*** -import java.util.Vector; - -public class Playbook ***REMOVED*** - - private String hostGroupName; - - private List roles; - - private String remote_user; - - public Playbook() ***REMOVED*** - this.roles = new Vector<>(); - ***REMOVED*** - - public void addRole(String role) ***REMOVED*** - roles.add(role); - ***REMOVED*** - - public void applyTo(String hostGroupName) ***REMOVED*** - this.hostGroupName = hostGroupName; - ***REMOVED*** - - public String getHostGroupName() ***REMOVED*** - return hostGroupName; - ***REMOVED*** - - public List getRoles() ***REMOVED*** - return new Vector<>(roles); - ***REMOVED*** - -public String getRemote_user() ***REMOVED*** - return remote_user; -***REMOVED*** - -public void setRemote_user(String remote_user) ***REMOVED*** - this.remote_user = remote_user; -***REMOVED*** - -public void setHostGroupName(String hostGroupName) ***REMOVED*** - this.hostGroupName = hostGroupName; -***REMOVED*** - -public void setRoles(List roles) ***REMOVED*** - this.roles = roles; -***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java deleted file mode 100644 index 1876ea7..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class Role ***REMOVED*** - - /** - * The name of the role - */ - private String name; - - private Collection tasks; - - private Collection meta; - - public Role() ***REMOVED*** - this.tasks = new Vector<>(); - this.meta = new Vector<>(); - ***REMOVED*** - - public Role(String name) ***REMOVED*** - this(); - this.name = name; - ***REMOVED*** - - public void addTaskFile(RoleFile tf) ***REMOVED*** - this.tasks.add(tf); - ***REMOVED*** - - public void addMeta(RoleFile tf) ***REMOVED*** - this.meta.add(tf); - ***REMOVED*** - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public Collection getTaskFiles() ***REMOVED*** - return new Vector<>(this.tasks); - ***REMOVED*** - - public Collection getMeta() ***REMOVED*** - return new Vector<>(this.meta); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java deleted file mode 100644 index ba903a9..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -public class RoleFile ***REMOVED*** - - /** - * The path to the file, starting from the role root - */ - private String path; - - /** - * The name of the task file - */ - private String name; - - /** - * The content of the task file - * @return - */ - private String content; - - public RoleFile() ***REMOVED*** - ***REMOVED*** - - public RoleFile(String name, String content) ***REMOVED*** - this(); - this.setName(name); - this.setContent(content); - ***REMOVED*** - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public String getContent() ***REMOVED*** - return content; - ***REMOVED*** - - public void setContent(String content) ***REMOVED*** - this.content = content; - ***REMOVED*** - - public String getPath() ***REMOVED*** - return path; - ***REMOVED*** - - public void setPath(String path) ***REMOVED*** - this.path = path; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java deleted file mode 100644 index e1f0ad5..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ /dev/null @@ -1,276 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; - -import java.io.File; -import java.io.FileOutputStream; -***REMOVED*** -import java.io.PrintStream; -import java.util.Collection; -***REMOVED*** -import java.util.Map; -***REMOVED*** -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.UUID; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.AlgorithmPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CranDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomRoleManager; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.OSDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.StaticRoleManager; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; -***REMOVED*** - -public class AnsibleBridge ***REMOVED*** - private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); - - - /** - * The workdir for this service - */ - private String dpmRoot; - - public AnsibleBridge() ***REMOVED*** - this(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); - ***REMOVED*** - - public AnsibleBridge(String root) ***REMOVED*** - this.dpmRoot = root; - this.ensureServiceRoot(); - ***REMOVED*** - - private void ensureServiceRoot() ***REMOVED*** - ***REMOVED*** generate root - new File(dpmRoot).mkdirs(); - ***REMOVED*** 'template' is for template roles - this.getTemplatesDir().mkdirs(); - ***REMOVED*** 'static' is for custom roles - this.getCustomDir().mkdirs(); - ***REMOVED*** 'work' is for temporary working directories - this.getWorkDir().mkdirs(); - ***REMOVED*** - - private File getWorkDir() ***REMOVED*** - return new File(this.dpmRoot, "work"); - ***REMOVED*** - - private File getTemplatesDir() ***REMOVED*** - return new File(this.dpmRoot, "templates"); - ***REMOVED*** - - private File getCustomDir() ***REMOVED*** - return new File(this.dpmRoot, "custom"); - ***REMOVED*** - - public AnsibleWorker createWorker() ***REMOVED*** - File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); - AnsibleWorker worker = new AnsibleWorker(workerRoot); - return worker; - ***REMOVED*** - - /** - * Groups hosts by domain and algorithm sets - * @param clusters - */ - public void printInventoryByDomainAndSets(Collection clusters) ***REMOVED*** - Map> inventory = new TreeMap<>(); - for(Cluster cluster:clusters) ***REMOVED*** - for(AlgorithmSet as:cluster.getAlgorithmSets()) ***REMOVED*** - String asName = as.getName(); - for(Host h:cluster.getHosts()) ***REMOVED*** - String domain = h.getDomain().getName(); - String key = String.format("[%s@%s]", asName, domain); - Set hosts = inventory.get(key); - if(hosts==null) ***REMOVED*** - hosts = new TreeSet<>(new HostComparator()); - inventory.put(key, hosts); - ***REMOVED*** - hosts.add(h); - ***REMOVED*** - - ***REMOVED*** - ***REMOVED*** - for(String key:inventory.keySet()) ***REMOVED*** - System.out.println(key); - Collection hosts = inventory.get(key); - for(Host h:hosts) ***REMOVED*** - System.out.println(h.getName()+"."+h.getDomain().getName()); - ***REMOVED*** - System.out.println(); - ***REMOVED*** - ***REMOVED*** - - /** - * Groups hosts by algorithm sets only - * @param clusters - */ - public void printInventoryBySets(Collection clusters) ***REMOVED*** - Map> inventory = new TreeMap<>(); - for (Cluster cluster : clusters) ***REMOVED*** - for (AlgorithmSet as : cluster.getAlgorithmSets()) ***REMOVED*** - String asName = as.getName(); - for (Host h : cluster.getHosts()) ***REMOVED*** - String key = String.format("[%s]", asName); - Set hosts = inventory.get(key); - if (hosts == null) ***REMOVED*** - hosts = new TreeSet<>(new HostComparator()); - inventory.put(key, hosts); - ***REMOVED*** - hosts.add(h); - ***REMOVED*** - - ***REMOVED*** - ***REMOVED*** - for (String key : inventory.keySet()) ***REMOVED*** - System.out.println(key); - Collection hosts = inventory.get(key); - for (Host h : hosts) ***REMOVED*** - System.out.println(h.getName()+"."+h.getDomain().getName()); - ***REMOVED*** - System.out.println(); - ***REMOVED*** - ***REMOVED*** - - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException ***REMOVED*** - - - return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString()); - ***REMOVED*** - - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid) throws IOException ***REMOVED*** - AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); - - - List algoRoles = new Vector<>(); - - ***REMOVED*** add algorithms and dependencies to the worker - for (Algorithm a : as.getAlgorithms()) ***REMOVED*** - for (Role r : this.generateRoles(a)) ***REMOVED*** - algoRoles.add(r); - worker.addRole(r); - ***REMOVED*** - for (Dependency d : a.getDependencies()) ***REMOVED*** - for (Role r : this.generateRoles(d)) ***REMOVED*** - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - - ***REMOVED*** add static roles - for(Role r:this.getStaticRoleManager().getStaticRoles()) ***REMOVED*** - worker.addRole(r); - ***REMOVED*** - - ***REMOVED*** generate the inventory - Inventory inventory = new Inventory(); - for (Host h : cluster.getHosts()) ***REMOVED*** - AnsibleHost ah = new AnsibleHost(h.getName()); - inventory.addHost(ah, "universe"); - inventory.addHost(ah, "d4science"); - ***REMOVED*** - worker.setInventory(inventory); - - ***REMOVED*** generate the playbook - Playbook playbook = new Playbook(); - playbook.setRemote_user("root"); - playbook.applyTo("universe"); - for(Role r:algoRoles) ***REMOVED*** - ***REMOVED*** add only 'add' roles - if(!r.getName().endsWith("remove")) ***REMOVED*** - playbook.addRole(r.getName()); - ***REMOVED*** - ***REMOVED*** - - worker.setPlaybook(playbook); - - ***REMOVED*** execute and save log locally - PrintStream console = System.out; - File path = new File(worker.getWorkdir() + File.separator + "logs"); - path.mkdirs(); - File n = new File(path + File.separator + worker.getWorkerId()); - FileOutputStream fos = new FileOutputStream(n); - PrintStream ps = new PrintStream(fos); - System.setOut(ps); - System.setErr(ps); - ***REMOVED***System.setErr(console); - - worker.apply(); - ***REMOVED***System.setOut(console); - ***REMOVED***worker.apply(); - System.out.println("Log stored to to " + n.getAbsolutePath()); - - ***REMOVED*** destroy the worker - worker.destroy(); - return worker; - ***REMOVED*** - - - - private TemplateManager getTemplateManager() ***REMOVED*** - return new TemplateManager(this.dpmRoot+"/templates"); - ***REMOVED*** - - private CustomRoleManager getCustomRoleManager() ***REMOVED*** - return new CustomRoleManager(this.dpmRoot+"/custom"); - ***REMOVED*** - - private StaticRoleManager getStaticRoleManager() ***REMOVED*** - return new StaticRoleManager(this.dpmRoot+"/static"); - ***REMOVED*** - - /** - * Generate all roles for this dependency - * @param d - */ - public Collection generateRoles(Dependency d) ***REMOVED*** - Collection roles = new Vector<>(); - - - if("os".equalsIgnoreCase(d.getType())) ***REMOVED*** - OSDependencyPackage pkg = new OSDependencyPackage(d); - if(pkg!=null) ***REMOVED*** - roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** - - ***REMOVED*** else if("custom".equalsIgnoreCase(d.getType())) ***REMOVED*** - CustomDependencyPackage pkg = new CustomDependencyPackage(d); - if(pkg!=null) ***REMOVED*** - roles.addAll(pkg.getRoles(this.getCustomRoleManager())); - ***REMOVED*** - ***REMOVED*** - - else if("github".equalsIgnoreCase(d.getType())) ***REMOVED*** - CranDependencyPackage pkg = new CranDependencyPackage(d); - if(pkg!=null) ***REMOVED*** - roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** - ***REMOVED*** - else if("cran".equalsIgnoreCase(d.getType())) ***REMOVED*** - CranDependencyPackage pkg = new CranDependencyPackage(d); - if(pkg!=null) ***REMOVED*** - roles.addAll(pkg.getRoles(this.getTemplateManager())); - ***REMOVED*** - - - ***REMOVED*** - return roles; - ***REMOVED*** - - public Collection generateRoles(Algorithm a) ***REMOVED*** - AlgorithmPackage pkg = new AlgorithmPackage(a); - return pkg.getRoles(this.getTemplateManager()); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java deleted file mode 100644 index eda7460..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -***REMOVED*** -import java.io.PrintWriter; - -import org.apache.commons.io.IOUtils; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.HostGroup; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleSerializeHelper ***REMOVED*** - - public static void serialize(Inventory inventory, File inventoryFile) throws IOException ***REMOVED*** - String out = ""; - for(HostGroup hg:inventory.getHostGroups()) ***REMOVED*** - out+=String.format("[%s]\n", hg.getName()); - for(AnsibleHost h:hg.getHosts()) ***REMOVED*** - out+=h.getName()+"\n"; - ***REMOVED*** - out+="\n"; - ***REMOVED*** - out = out.trim(); - serialize(out, inventoryFile); - ***REMOVED*** - - public static void serialize(Playbook playbook, File playbookFile) throws IOException ***REMOVED*** - String out = "- hosts: " + playbook.getHostGroupName() + "\n"; - out += " remote_user: "+playbook.getRemote_user()+"\n"; - out+=" roles:\n"; - for(String r:playbook.getRoles()) ***REMOVED*** - out+=" - " + r+"\n"; - ***REMOVED*** - out+=" vars:\n"; - out+=" os_package_state: present\n"; - out = out.trim(); - serialize(out, playbookFile); - ***REMOVED*** - - public static void serializeRole(Role r, File dir) throws IOException ***REMOVED*** - ***REMOVED*** create root - File root = new File(dir, r.getName()); - root.mkdirs(); - - ***REMOVED*** create tasks - if(r.getTaskFiles().size()>0) ***REMOVED*** - File tasks = new File(root, "tasks"); - tasks.mkdirs(); - for(RoleFile tf: r.getTaskFiles()) ***REMOVED*** - serializeTask(tf, tasks); - ***REMOVED*** - ***REMOVED*** - - ***REMOVED*** create meta - if(r.getMeta().size()>0) ***REMOVED*** - File meta = new File(root, "meta"); - meta.mkdirs(); - for(RoleFile tf: r.getMeta()) ***REMOVED*** - serializeTask(tf, meta); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - - public static void serializeTask(RoleFile tf, File dir) throws IOException ***REMOVED*** - File f = new File(dir, tf.getName()); - serialize(tf.getContent().trim(), f); - ***REMOVED*** - - public static void serialize(String s, File f) throws IOException ***REMOVED*** - PrintWriter out = new PrintWriter(f); - out.println(s); - out.close(); - ***REMOVED*** - - public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException ***REMOVED*** - Role out = new Role(); - out.setName(roleDir.getName()); - - if(!roleDir.exists()) ***REMOVED*** - throw new FileNotFoundException(); - ***REMOVED*** - - try ***REMOVED*** - File tasksDir = new File(roleDir, "tasks"); - if(tasksDir.exists()) ***REMOVED*** - for(File main:tasksDir.listFiles()) ***REMOVED*** - String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); - RoleFile tf = new RoleFile(main.getName(), content); - tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); - out.addTaskFile(tf); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch(FileNotFoundException e) ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** - - try ***REMOVED*** - File metaDir = new File(roleDir, "meta"); - if(metaDir.exists()) ***REMOVED*** - for(File main:metaDir.listFiles()) ***REMOVED*** - String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); - RoleFile tf = new RoleFile(main.getName(), content); - tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); - out.addMeta(tf); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** catch(FileNotFoundException e) ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** - - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java deleted file mode 100644 index 64f0338..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Vector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class AlgorithmPackage ***REMOVED*** - - private Algorithm algorithm; - - public AlgorithmPackage(Algorithm a) ***REMOVED*** - this.algorithm = a; - ***REMOVED*** - - protected Map getDictionary(Algorithm a) ***REMOVED*** - Map out = new HashMap(); - out.put("name", a.getName()); - out.put("category", a.getCategory()); - out.put("class", a.getClazz()); - out.put("atype", a.getAlgorithmType()); - out.put("skipjava", a.getSkipJava()); - out.put("vre", ScopeProvider.instance.get()); - ***REMOVED***out.put("vre", "FAKE_VRE"); - out.put("packageurl", a.getPackageURL()); - out.put("description", a.getDescription()); - String deps = ""; - - for(Dependency d:a.getDependencies()) ***REMOVED*** - deps+=String.format("- ***REMOVED*** role: %s ***REMOVED***\n", d.getType()+"-"+d.getName().replaceAll("/", "-")); - ***REMOVED*** - deps = deps.trim(); - out.put("dependencies", deps); - return out; - ***REMOVED*** - - protected Algorithm getAlgorithm() ***REMOVED*** - return this.algorithm; - ***REMOVED*** - - public Collection getRoles(TemplateManager tm) ***REMOVED*** - Collection out = new Vector<>(); - for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update" - String roleName = "gcube-algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode); - try ***REMOVED*** - ***REMOVED*** find template - Role template = tm.getRoleTemplate("gcube-algorithm-" + mode); - ***REMOVED*** - if(template!=null) ***REMOVED*** - Map dictionary = this.getDictionary(this.getAlgorithm()); - Role r = tm.fillRoleTemplate(template, dictionary); - r.setName(roleName); - out.add(r); - ***REMOVED*** else ***REMOVED*** - System.out.println("WARNING: template is null"); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); - System.out.println("WARNING: no template found for " + roleName); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java deleted file mode 100644 index a040404..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class CranDependencyPackage extends DependencyPackage ***REMOVED*** - - public CranDependencyPackage(Dependency d) ***REMOVED*** - super(d); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java deleted file mode 100644 index 4bd4b65..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.NoSuchElementException; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class CustomDependencyPackage extends DependencyPackage ***REMOVED*** - - public CustomDependencyPackage(Dependency d) ***REMOVED*** - super(d); - ***REMOVED*** - - private String getCustomRepositoryLocation(String ansibleRoot) ***REMOVED*** - return ansibleRoot+"/custom"; - ***REMOVED*** - - /* - public void serializeTo(String ansibleRoot) ***REMOVED*** - for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED*** - ***REMOVED*** look for roles in the 'custom' repository - try ***REMOVED*** - ***REMOVED*** role name - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - ***REMOVED*** look for the custom role - File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName); - System.out.println("** CUSTOM ** " + src); - if(src.exists()) ***REMOVED*** - ***REMOVED*** do copy - System.out.println("copying CUSTOM role"); - File dest = new File(ansibleRoot+"/work/"+roleName); - FileUtils.copyDirectory(src, dest); - ***REMOVED*** - ***REMOVED*** catch(IOException e) ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - */ - - public Collection getRoles(CustomRoleManager crm) ***REMOVED*** - Collection out = new Vector<>(); -***REMOVED*** for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED*** - for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update" - ***REMOVED*** role name - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - try ***REMOVED*** - ***REMOVED*** look for custom role - Role role = crm.getRole(roleName); - if(role!=null) ***REMOVED*** - out.add(role); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); - System.out.println("WARNING: no custom role found for " + roleName); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java deleted file mode 100644 index c296a5c..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.io.File; -***REMOVED*** -import java.util.NoSuchElementException; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - -public class CustomRoleManager ***REMOVED*** - - private String root; - - public CustomRoleManager(String root) ***REMOVED*** - this.root = root; - ***REMOVED*** - - public String getRoot() ***REMOVED*** - return this.root; - ***REMOVED*** - - public Role getRole(String roleName) throws NoSuchElementException ***REMOVED*** - File f = new File(this.getRoot(), roleName); - try ***REMOVED*** - return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - ***REMOVED*** catch (IOException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); - throw new NoSuchElementException("unable to find " + roleName); - ***REMOVED*** - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java deleted file mode 100644 index 771afab..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class DependencyPackage ***REMOVED*** - - private Dependency dependency; - - public DependencyPackage(Dependency d) ***REMOVED*** - this.dependency = d; - ***REMOVED*** - - protected Map getDictionary(Dependency d) ***REMOVED*** - Map out = new HashMap(); - out.put("name", d.getName()); - out.put("type", d.getType()); - return out; - ***REMOVED*** - - protected Dependency getDependency() ***REMOVED*** - return this.dependency; - ***REMOVED*** - - public Collection getRoles(TemplateManager tm) ***REMOVED*** - Collection out = new Vector<>(); - for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update" - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName().replaceAll("/", "-")+("add".equals(mode) ? "" : "-"+mode); - try ***REMOVED*** - ***REMOVED*** find template - Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode); - ***REMOVED*** - if(template!=null) ***REMOVED*** - Map dictionary = this.getDictionary(this.getDependency()); - Role r = tm.fillRoleTemplate(template, dictionary); - r.setName(roleName); - out.add(r); - ***REMOVED*** else ***REMOVED*** - System.out.println("WARNING: template is null"); - ***REMOVED*** - ***REMOVED*** catch (NoSuchElementException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); - System.out.println("WARNING: no template found for " + roleName); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java deleted file mode 100644 index 4eaa7e2..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class OSDependencyPackage extends DependencyPackage ***REMOVED*** - - public OSDependencyPackage(Dependency d) ***REMOVED*** - super(d); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java deleted file mode 100644 index f333e76..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - - import java.io.File; -***REMOVED*** -import java.util.Collection; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - - public class StaticRoleManager ***REMOVED*** - - private String root; - - public StaticRoleManager(String root) ***REMOVED*** - this.root = root; - ***REMOVED*** - - public String getRoot() ***REMOVED*** - return this.root; - ***REMOVED*** - - public Collection getStaticRoles() ***REMOVED*** - Collection out = new Vector<>(); - for(File f: new File(this.getRoot()).listFiles()) ***REMOVED*** - try ***REMOVED*** - out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f)); - ***REMOVED*** catch(IOException e) ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - - - ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java deleted file mode 100644 index 1781bff..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.io.File; -***REMOVED*** -import java.util.Map; -import java.util.NoSuchElementException; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; -import org.stringtemplate.v4.ST; - -public class TemplateManager ***REMOVED*** - - private String root; - - public TemplateManager(String root) ***REMOVED*** - this.root = root; - ***REMOVED*** - - public String getTemplateRoot() ***REMOVED*** - return this.root; - ***REMOVED*** - - /** - * Read the given template - * @param templateName - * @return - * @throws IOException - */ -***REMOVED*** private String readTemplate(String templateName) throws IOException ***REMOVED*** -***REMOVED*** File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); -***REMOVED*** System.out.println("looking for file " + templateFile.getName()); -***REMOVED*** String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8"); -***REMOVED*** return out; -***REMOVED*** ***REMOVED*** - - /** - * Return the content of the given template - * @param templateName - * @return - * @throws NoSuchElementException if no such template exists - */ -***REMOVED*** public String getTemplate(String templateName) throws NoSuchElementException ***REMOVED*** -***REMOVED*** String template = null; -***REMOVED*** try ***REMOVED*** -***REMOVED*** template = this.readTemplate(templateName); -***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED*** -***REMOVED*** throw new NoSuchElementException(); -***REMOVED*** ***REMOVED*** -***REMOVED*** return template; -***REMOVED*** ***REMOVED*** - - public Role fillRoleTemplate(Role template, Map dictionary) ***REMOVED*** - Role out = new Role(); - out.setName(template.getName()); - for(RoleFile tf:template.getTaskFiles()) ***REMOVED*** - out.addTaskFile(this.fillTaskTemplate(tf, dictionary)); - ***REMOVED*** - for(RoleFile tf:template.getMeta()) ***REMOVED*** - out.addMeta(this.fillTaskTemplate(tf, dictionary)); - ***REMOVED*** - return out; - ***REMOVED*** - - private RoleFile fillTaskTemplate(RoleFile template, Map dictionary) ***REMOVED*** - RoleFile out = new RoleFile(); - out.setName(template.getName()); - out.setContent(this.fillTemplate(template.getContent(), dictionary)); - return out; - ***REMOVED*** - - private String fillTemplate(String template, Map dictionary) ***REMOVED*** - if (template != null) ***REMOVED*** - ST t = new ST(template); - for (String key : dictionary.keySet()) ***REMOVED*** - t.add(key, dictionary.get(key)); - ***REMOVED*** - String output = t.render(); - return output; - ***REMOVED*** - return template; - ***REMOVED*** - - public Role getRoleTemplate(String roleName) throws NoSuchElementException ***REMOVED*** - File f = new File(this.getTemplateRoot(), roleName); - try ***REMOVED*** - return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - ***REMOVED*** catch (IOException e) ***REMOVED*** -***REMOVED*** e.printStackTrace(); - throw new NoSuchElementException("unable to find " + roleName); - ***REMOVED*** - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java deleted file mode 100644 index 3b1bbab..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.clients; - -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - -import java.util.Collection; -***REMOVED*** -import java.util.Vector; - -import org.gcube.common.resources.gcore.ServiceEndpoint; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; - -public class ISClient ***REMOVED*** - - /** - * Return the list of hosts (dataminers) in a given VRE - * - * @param vreName - * @return - */ - public Collection listDataminersInVRE() ***REMOVED*** - - boolean remote = false; - - if (!remote) ***REMOVED*** - Collection out = new Vector<>(); - Host h = new Host(); - ***REMOVED***h.setName("bb-dataminer.res.eng.it"); - ***REMOVED***h.setName("vm101.ui.savba.sk"); - h.setName("dataminer1-devnext.d4science.org"); - out.add(h); - return out; - ***REMOVED*** else ***REMOVED*** - - SimpleQuery query = queryFor(ServiceEndpoint.class); - - ***REMOVED***old version - ***REMOVED***query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'") - ***REMOVED***.addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); - - query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'"); - - DiscoveryClient client = clientFor(ServiceEndpoint.class); - - List resources = client.submit(query); - - Collection out = new Vector<>(); - for (ServiceEndpoint r : resources) ***REMOVED*** - Host h = new Host(); - h.setName(r.profile().runtime().hostedOn()); - out.add(h); - ***REMOVED*** - return out; - ***REMOVED*** - ***REMOVED*** - - public static void main(String[] args) ***REMOVED*** - ISClient a = new ISClient(); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(a.listDataminersInVRE()); -***REMOVED*** -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java deleted file mode 100644 index c4162f6..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Action ***REMOVED*** - - private String name; - - private String description; - - private String script; - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public String getDescription() ***REMOVED*** - return description; - ***REMOVED*** - - public void setDescription(String description) ***REMOVED*** - this.description = description; - ***REMOVED*** - - public String getScript() ***REMOVED*** - return script; - ***REMOVED*** - - public void setScript(String script) ***REMOVED*** - this.script = script; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java deleted file mode 100644 index e53029b..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class Algorithm ***REMOVED*** - - private String name; - - private String description; - - private String category; - - private String clazz; - - private String algorithmType; - - private String skipJava; - - private String packageURL; - - private Collection actions; - - private Collection dependencies; - - public Algorithm() ***REMOVED*** - this.actions = new Vector<>(); - this.dependencies = new Vector<>(); - Dependency p = new Dependency(); - ***REMOVED*** - - public void addDependency(Dependency dep) ***REMOVED*** - this.dependencies.add(dep); - ***REMOVED*** - - public void addAction(Action action) ***REMOVED*** - this.actions.add(action); - ***REMOVED*** - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public String getDescription() ***REMOVED*** - return description; - ***REMOVED*** - - public void setDescription(String description) ***REMOVED*** - this.description = description; - ***REMOVED*** - - public String getCategory() ***REMOVED*** - return category; - ***REMOVED*** - - public void setCategory(String category) ***REMOVED*** - this.category = category; - ***REMOVED*** - - public Collection getActions() ***REMOVED*** - return actions; - ***REMOVED*** - - public Collection getDependencies() ***REMOVED*** - return dependencies; - ***REMOVED*** - - public void setDependencies(Collection deps) ***REMOVED*** - this.dependencies = deps; - ***REMOVED*** - - public String toString() ***REMOVED*** - String out = "Algorithm: " + this.getName()+"\n"; - out+=" Class Name: " + this.getClazz()+"\n"; - out+=" Description: " + this.getDescription()+"\n"; - out+=" Dependencies: " + this.getDependencies()+"\n"; - return out; - ***REMOVED*** - - public String getClazz() ***REMOVED*** - return clazz; - ***REMOVED*** - - public void setClazz(String clazz) ***REMOVED*** - this.clazz = clazz; - ***REMOVED*** - - public String getPackageURL() ***REMOVED*** - return packageURL; - ***REMOVED*** - - public void setPackageURL(String packageURL) ***REMOVED*** - this.packageURL = packageURL; - ***REMOVED*** - - public String getAlgorithmType() ***REMOVED*** - return algorithmType; - ***REMOVED*** - - public void setAlgorithmType(String algorithmType) ***REMOVED*** - this.algorithmType = algorithmType; - ***REMOVED*** - - public String getSkipJava() ***REMOVED*** - return skipJava; - ***REMOVED*** - - public void setSkipJava(String skipJava) ***REMOVED*** - this.skipJava = skipJava; - ***REMOVED*** - - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java deleted file mode 100644 index d209f89..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class AlgorithmSet ***REMOVED*** - - private String name; - - private Collection algorithms; - - public AlgorithmSet() ***REMOVED*** - this.algorithms = new Vector<>(); - ***REMOVED*** - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public Collection getAlgorithms() ***REMOVED*** - return new Vector<>(algorithms); - ***REMOVED*** - - public void addAlgorithm(Algorithm algoritm) ***REMOVED*** - this.algorithms.add(algoritm); - ***REMOVED*** - - public Boolean hasAlgorithm(Algorithm algorithm) ***REMOVED*** - for (Algorithm a : this.algorithms) ***REMOVED*** - if (a.getName().equals(algorithm.getName())) ***REMOVED*** - return true; - ***REMOVED*** - ***REMOVED*** - return false; - ***REMOVED*** - - public String toString() ***REMOVED*** - String out = "ALGOSET: " + this.name + "\n"; - for(Algorithm a:this.algorithms) ***REMOVED*** - out+=a+"\n"; - ***REMOVED*** - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java deleted file mode 100644 index 9432df4..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class Cluster ***REMOVED*** - - /** - * The set of hosts belonging to the cluster. - */ - private Collection hosts; - - /** - * A name for this cluster. - */ - private String name; - - /** - * A description of this cluster. - */ - private String description; - - /** - * The set of algorithms deployed on this cluster (i.e. on all its hosts) - */ - private Collection algoSets; - - public Cluster() ***REMOVED*** - this.hosts = new Vector<>(); - this.algoSets = new Vector<>(); - ***REMOVED*** - - public void addAlgorithmSet(AlgorithmSet set) ***REMOVED*** - this.algoSets.add(set); - ***REMOVED*** - - public void addHost(Host host) ***REMOVED*** - this.hosts.add(host); - ***REMOVED*** - - public Collection getHosts() ***REMOVED*** - return hosts; - ***REMOVED*** - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public String getDescription() ***REMOVED*** - return description; - ***REMOVED*** - - public void setDescription(String description) ***REMOVED*** - this.description = description; - ***REMOVED*** - - public Collection getAlgorithmSets() ***REMOVED*** - return algoSets; - ***REMOVED*** - - public String toString() ***REMOVED*** - String out = "Cluster: "+this.name+"\n"; - for(Host h:this.getHosts()) ***REMOVED*** - out+=" "+h+"\n"; - ***REMOVED*** - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java deleted file mode 100644 index b623c63..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Dependency ***REMOVED*** - - private String name; - - private String type; - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public String getType() ***REMOVED*** - return type; - ***REMOVED*** - - public void setType(String type) ***REMOVED*** - this.type = type; - ***REMOVED*** - - public String toString() ***REMOVED*** - return this.type+":"+this.name; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java deleted file mode 100644 index a7fc3ee..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Domain ***REMOVED*** - - private String name; - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java deleted file mode 100644 index 63d3569..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Host ***REMOVED*** - - private String name; - - private Domain domain; - - public Host() ***REMOVED*** - ***REMOVED*** - - public String getFullyQualifiedName() ***REMOVED*** - if(this.domain!=null && this.domain.getName()!=null) - return this.getName()+"."+this.getDomain().getName(); - else - return this.getName(); - ***REMOVED*** - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public void setName(String name) ***REMOVED*** - this.name = name; - ***REMOVED*** - - public Domain getDomain() ***REMOVED*** - return domain; - ***REMOVED*** - - public void setDomain(Domain domain) ***REMOVED*** - this.domain = domain; - ***REMOVED*** - - public String toString() ***REMOVED*** - return this.name + "@" + this.domain; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java deleted file mode 100644 index c3f3894..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -***REMOVED*** - -public class AlgorithmComparator implements Comparator ***REMOVED*** - - @Override - public int compare(Algorithm a1, Algorithm a2) ***REMOVED*** - return a1.getName().compareTo(a2.getName()); - ***REMOVED*** - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java deleted file mode 100644 index 77c27c5..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class DependencyComparator implements Comparator ***REMOVED*** - - @Override - public int compare(Dependency a1, Dependency a2) ***REMOVED*** - int out = a1.getType().compareTo(a2.getType()); - if(out!=0) - return out; - return a1.getName().compareTo(a2.getName()); - ***REMOVED*** - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java deleted file mode 100644 index c57f9ca..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; - -public class HostComparator implements Comparator ***REMOVED*** - - @Override - public int compare(Host h1, Host h2) ***REMOVED*** - int out = h1.getDomain().getName().compareTo(h2.getDomain().getName()); - if(out!=0) - return out; - return h1.getName().compareTo(h2.getName()); - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java deleted file mode 100644 index 6ee5ada..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java +++ /dev/null @@ -1,103 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.process; - -import java.util.StringTokenizer; - -public class AddAlgorithmCommand ***REMOVED*** - - private String command; - private String name; - private String category; - private String clazz; - private String scope; - private String algorithmType; - private String skipJava; - private String url; - private String description; - - public AddAlgorithmCommand(String cmd) ***REMOVED*** - StringTokenizer st = new StringTokenizer(cmd, " "); - if (st.hasMoreElements()) - command = st.nextToken(); - if (st.hasMoreElements()) - name = st.nextToken(); - if (st.hasMoreElements()) - category = st.nextToken(); - if (st.hasMoreElements()) - clazz = st.nextToken(); - if (st.hasMoreElements()) - scope = st.nextToken(); - if (st.hasMoreElements()) - algorithmType = st.nextToken(); - if (st.hasMoreElements()) - skipJava = st.nextToken(); - if (st.hasMoreElements()) - url = st.nextToken(); - - String d = ""; - while (st.hasMoreElements()) - d = d + st.nextToken() + " "; - this.setDescription(d); - - ***REMOVED*** - - public void setDescription(String d) ***REMOVED*** - if(d!=null) ***REMOVED*** - d = d.trim(); - if(d.startsWith("\"") && d.endsWith("\"")) ***REMOVED*** - d = d.substring(1, d.length()-1).trim(); - ***REMOVED*** - ***REMOVED*** - this.description = d; - ***REMOVED*** - - public String getCommand() ***REMOVED*** - return command; - ***REMOVED*** - - public String getName() ***REMOVED*** - return name; - ***REMOVED*** - - public String getCategory() ***REMOVED*** - return category; - ***REMOVED*** - - public String getClazz() ***REMOVED*** - return clazz; - ***REMOVED*** - - public String getVRE() ***REMOVED*** - return scope; - ***REMOVED*** - - public String getAlgorithmType() ***REMOVED*** - return algorithmType; - ***REMOVED*** - - public String getSkipjava() ***REMOVED*** - return skipJava; - ***REMOVED*** - - public String getUrl() ***REMOVED*** - return url; - ***REMOVED*** - - public String getDescription() ***REMOVED*** - return description; - ***REMOVED*** - - public String toString() ***REMOVED*** - String out = ""; - out += String.format("%-12s: %s\n", "command", command); - out += String.format("%-12s: %s\n", "algo name", name); - out += String.format("%-12s: %s\n", "category", category); - out += String.format("%-12s: %s\n", "class", clazz); - out += String.format("%-12s: %s\n", "scope", scope); - out += String.format("%-12s: %s\n", "algo type", algorithmType); - out += String.format("%-12s: %s\n", "skip java", skipJava); - out += String.format("%-12s: %s\n", "url", url); - out += String.format("%-12s: %s\n", "description", this.description); - return out; - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java deleted file mode 100644 index 62dcb06..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ /dev/null @@ -1,290 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.process; - -***REMOVED*** -import java.io.InputStream; -***REMOVED*** -import java.util.HashMap; -***REMOVED*** -import java.util.Map; -import java.util.Vector; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import org.apache.commons.lang.StringEscapeUtils; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class AlgorithmPackageParser ***REMOVED*** - - /** - * The name of the file containing algorithm metadata. Expected in the root - * directory of the package. - */ - private static final String METADATA_FILE_NAME = "Info.txt"; - - private static final String METADATA_ALGORITHM_NAME = "Algorithm Name"; - - private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description"; - - private static final String METADATA_CLASS_NAME = "Class Name"; - - private static final String METADATA_PACKAGES = "Packages"; - - private static final String METADATA_KEY_VALUE_SEPARATOR = ":"; - - private static final int BUFFER_SIZE = 4096; - - /** - * Given an URL to an algorithm package, create an Algorithm object with its - * metadata. Metadata are extracted from the 'info.txt' file, if any, in the - * package. - * - * @param url - * @return An Algorithm object or null if no 'info.txt' is found in the - * package. - * @throws IOException - */ - public Algorithm parsePackage(String url) throws IOException ***REMOVED*** - String packageMetadata = this.getPackageMetadata(url); - if (packageMetadata == null) ***REMOVED*** - System.out.println("WARNING: No metadata found for " + url); - return null; - ***REMOVED*** else ***REMOVED*** - Map> parsedMetadata = this.parseMetadata(packageMetadata); - Algorithm a = this.createAlgorithm(parsedMetadata); - a.setPackageURL(url); - return a; - ***REMOVED*** - ***REMOVED*** - - /** - * Extract the content of the metadata file from the package. - * - * @param url - * @return - * @throws IOException - */ - private String getPackageMetadata(String url) throws IOException ***REMOVED*** - InputStream is = new URL(url).openStream(); - ZipInputStream zipIs = new ZipInputStream(is); - ZipEntry entry = zipIs.getNextEntry(); - String out = null; - while (entry != null) ***REMOVED*** - if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) ***REMOVED*** - out = this.getEntryContent(zipIs); - break; - ***REMOVED*** - entry = zipIs.getNextEntry(); - ***REMOVED*** - is.close(); - zipIs.close(); - return out; - ***REMOVED*** - - /** - * Read the content of a zip entry and place it in a string. - * @param zipIn - * @return - * @throws IOException - */ - private String getEntryContent(ZipInputStream zipIn) throws IOException ***REMOVED*** - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = zipIn.read(buffer)) != -1) ***REMOVED*** - s.append(new String(buffer, 0, read)); - ***REMOVED*** - return s.toString(); - ***REMOVED*** - - /** - * Parse the content of the metadata file and create a key+multivalue map. - * @param metadata - * @return - */ - private Map> parseMetadata(String metadata) ***REMOVED*** - Map> out = new HashMap>(); - String[] lines = metadata.split("\n"); - - String key = null; - String value = null; - - for (String line : lines) ***REMOVED*** - ***REMOVED*** skip empty lines - if (line.trim().isEmpty()) ***REMOVED*** - continue; - ***REMOVED*** - ***REMOVED*** scan lines one by one, looking for key and values - String[] parts = line.split(METADATA_KEY_VALUE_SEPARATOR); - if (parts.length > 1) ***REMOVED*** - ***REMOVED*** key and value on the same line - key = parts[0].trim(); - value = line.substring(parts[0].length() + 1).trim(); - ***REMOVED*** else if (parts.length == 1) ***REMOVED*** - ***REMOVED*** either a key or a value - if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) ***REMOVED*** - ***REMOVED*** key - key = parts[0].trim(); - value = null; - ***REMOVED*** else ***REMOVED*** - ***REMOVED*** value - value = line.trim(); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** add key+value to the map - if (key != null && value != null) ***REMOVED*** - List values = out.get(key); - if (values == null) ***REMOVED*** - values = new Vector<>(); - out.put(key, values); - ***REMOVED*** - values.add(value); - System.out.println(key + METADATA_KEY_VALUE_SEPARATOR + " " + values); - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - /** - * Create an Algorithm starting from its metadata - * @param metadata - * @return - */ -***REMOVED*** private Algorithm createAlgorithm(Map> metadata) ***REMOVED*** -***REMOVED*** Algorithm out = new Algorithm(); -***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); -***REMOVED*** out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); -***REMOVED*** out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); -***REMOVED*** List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); -***REMOVED*** if (dependencies != null) ***REMOVED*** -***REMOVED*** for (String pkg : dependencies) ***REMOVED*** -***REMOVED*** Dependency dep = new Dependency(); -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** out.addDependency(dep); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return out; -***REMOVED*** ***REMOVED*** - - - private Algorithm createAlgorithm(Map> metadata) ***REMOVED*** - Algorithm out = new Algorithm(); - out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); - out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); - out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); - ***REMOVED***List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); - - - List rdependencies = extractMultipleValues(metadata, "cran"); - if (rdependencies != null) ***REMOVED*** - for (String pkg : rdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("cran"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - - List defdependencies = extractMultipleValues(metadata, "Packages"); - if (defdependencies != null) ***REMOVED*** - for (String pkg : defdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - List osdependencies = extractMultipleValues(metadata, "os"); - if (osdependencies != null) ***REMOVED*** - for (String pkg : osdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - - - List gitdependencies = extractMultipleValues(metadata, "github"); - if (gitdependencies != null) ***REMOVED*** - for (String pkg : gitdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("github"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - - - List cdependencies = extractMultipleValues(metadata, "custom"); - if (cdependencies != null) ***REMOVED*** - for (String pkg : cdependencies) ***REMOVED*** - Dependency dep = new Dependency(); - - ***REMOVED***if (pkg.startsWith("os:"))***REMOVED*** - dep.setName(pkg); - dep.setType("custom"); - out.addDependency(dep); - ***REMOVED*** - ***REMOVED*** - - -***REMOVED*** if (pkg.startsWith("r:"))***REMOVED*** -***REMOVED*** ***REMOVED***String results = StringEscapeUtils.escapeJava(pkg); -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("cran"); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (pkg.startsWith("custom:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("custom"); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:"))***REMOVED*** -***REMOVED*** dep.setName(pkg); -***REMOVED*** dep.setType("os"); -***REMOVED*** ***REMOVED*** - - - - return out; - ***REMOVED*** - - - - - private static String extractSingleValue(Map> metadata, - String key) ***REMOVED*** - List l = metadata.get(key); - if (l != null && l.size() == 1) ***REMOVED*** - return l.get(0); - ***REMOVED*** else ***REMOVED*** - return null; - ***REMOVED*** - ***REMOVED*** - - - - - private static List extractMultipleValues( - Map> metadata, String key) ***REMOVED*** - List l = metadata.get(key); - if (l != null) ***REMOVED*** - return new Vector<>(l); - ***REMOVED*** else ***REMOVED*** - return null; - ***REMOVED*** - ***REMOVED*** - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java deleted file mode 100644 index 72e4fe6..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ /dev/null @@ -1,24 +0,0 @@ -***REMOVED*** - -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** - -***REMOVED*** - -public interface PoolManager ***REMOVED*** - - String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException; - - Algorithm extractAlgorithm(String url) throws IOException; - - String getLogById(String logId) throws IOException; - - void getLogId(Algorithm algo, String vre); - - String getScriptFromURL(URL logId) throws IOException; - - URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java deleted file mode 100644 index f45a587..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ /dev/null @@ -1,262 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.service; - -import java.io.BufferedReader; -import java.io.File; -***REMOVED*** -import java.io.InputStreamReader; -***REMOVED*** -***REMOVED*** -import java.net.URLConnection; -***REMOVED*** -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager; -***REMOVED*** -***REMOVED*** - -public class DataminerPoolManager implements PoolManager ***REMOVED*** - - - private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); - - -***REMOVED*** static Collection algorithms; -***REMOVED*** -***REMOVED*** static Collection sets; -***REMOVED*** -***REMOVED*** static ***REMOVED*** -***REMOVED*** algorithms = new Vector<>(); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public DataminerPoolManager() ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected -***REMOVED*** * on the pool. -***REMOVED*** */ -***REMOVED*** public void publishAlgorithm(Algorithm algorithm) ***REMOVED*** -***REMOVED*** algorithms.add(algorithm); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Re-deploy the given algorithm wherever it's installed -***REMOVED*** * -***REMOVED*** * @param algorithm -***REMOVED*** */ -***REMOVED*** /* -***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) ***REMOVED*** ***REMOVED*** TODO implement this ***REMOVED*** -***REMOVED*** */ -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Add the give algorithm to the given set -***REMOVED*** * -***REMOVED*** * @param algorithmId -***REMOVED*** * @param setId -***REMOVED*** */ -***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) ***REMOVED*** -***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); -***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName); -***REMOVED*** if (set != null && algorithm != null) ***REMOVED*** -***REMOVED*** set.addAlgorithm(algorithm); -***REMOVED*** this.updateClusters(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** /** -***REMOVED*** * Apply the given set of algorithms to the given cluster -***REMOVED*** * -***REMOVED*** * @param setId -***REMOVED*** * @param clusterId -***REMOVED*** */ -***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) ***REMOVED*** -***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName); -***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName); -***REMOVED*** if (set != null && cluster != null) ***REMOVED*** -***REMOVED*** cluster.addAlgorithmSet(set); -***REMOVED*** this.updateClusters(); -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) ***REMOVED*** -***REMOVED*** for (AlgorithmSet set : sets) ***REMOVED*** -***REMOVED*** if (name.equals(set.getName())) ***REMOVED*** -***REMOVED*** return set; -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** private Algorithm getAlgorithm(String name) ***REMOVED*** -***REMOVED*** for (Algorithm a : algorithms) ***REMOVED*** -***REMOVED*** if (name.equals(a.getName())) ***REMOVED*** -***REMOVED*** return a; -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** - - - - -***REMOVED*** -***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) ***REMOVED*** -***REMOVED*** new Thread() ***REMOVED*** -***REMOVED*** public void run() ***REMOVED*** -***REMOVED*** while (true) ***REMOVED*** -***REMOVED*** try ***REMOVED*** -***REMOVED*** addAlgorithmToVRE(algorithm, vre); -***REMOVED*** ***REMOVED*** catch (Exception e) ***REMOVED*** -***REMOVED*** ***REMOVED***log here -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED*** -***REMOVED*** ***REMOVED***.start(); -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** - -***REMOVED*** public String getLogId()***REMOVED*** -***REMOVED*** PrintStream console = System.out; -***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs"); -***REMOVED*** path.mkdirs(); -***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId()); -***REMOVED*** FileOutputStream fos = new FileOutputStream(n); -***REMOVED*** PrintStream ps = new PrintStream(fos); -***REMOVED*** System.setOut(ps); -***REMOVED*** worker.apply(); -***REMOVED*** System.setOut(console); -***REMOVED*** worker.apply(); -***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath()); -***REMOVED******REMOVED*** - - - - -***REMOVED*** public String getLogById(String id) throws IOException ***REMOVED*** -***REMOVED*** String strLine = null; -***REMOVED*** try***REMOVED*** -***REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id); -***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); -***REMOVED*** /* read log line by line */ -***REMOVED*** while ((strLine = br.readLine()) != null) ***REMOVED*** -***REMOVED*** /* parse strLine to obtain what you want */ -***REMOVED*** System.out.println (strLine); -***REMOVED*** ***REMOVED*** -***REMOVED*** br.close(); -***REMOVED*** ***REMOVED*** catch (Exception e) ***REMOVED*** -***REMOVED*** System.err.println("Error: " + e.getMessage()); -***REMOVED*** ***REMOVED*** -***REMOVED*** return strLine; -***REMOVED******REMOVED*** - - - public String getScriptFromURL(URL url) throws IOException ***REMOVED*** - if (url == null) ***REMOVED*** - return null; - ***REMOVED*** - URLConnection yc = url.openConnection(); - BufferedReader input = new BufferedReader(new InputStreamReader( - yc.getInputStream())); - String line; - StringBuffer buffer = new StringBuffer(); - while ((line = input.readLine()) != null) ***REMOVED*** - buffer.append(line + "\n"); - ***REMOVED*** - String bufferScript = buffer.substring(0, buffer.length()); - input.close(); - return bufferScript; - ***REMOVED*** - - - - - - /** - * Publish the given algorithm in the given VRE - * - * @param algorithmName - * @param vre - * - */ - public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException ***REMOVED*** - ***REMOVED*** create a fake algorithm set - final AlgorithmSet algoSet = new AlgorithmSet(); - algoSet.setName("fake"); - algoSet.addAlgorithm(algorithm); - final String uuid = UUID.randomUUID().toString(); - - new Thread(new Runnable() ***REMOVED*** -***REMOVED*** - public void run() ***REMOVED*** - ***REMOVED*** - try ***REMOVED*** - addAlgorithmsToVRE(algoSet, vre, uuid); - ***REMOVED*** catch (IOException e) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** -***REMOVED***).start(); - ***REMOVED***this line will execute immediately, not waiting for your task to complete - System.out.println(uuid); - return uuid; - ***REMOVED*** - - - - public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException***REMOVED*** - - File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs"); - path.mkdirs(); - File n = new File(path + File.separator +a); - ***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress(); - - return new File(n.getPath()).toURI().toURL(); - ***REMOVED*** - - - public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException ***REMOVED*** - - ***REMOVED*** create the cluster (dataminers in the vre) - Cluster cluster = new Cluster(); - for(Host h:new ISClient().listDataminersInVRE()) ***REMOVED*** - cluster.addHost(h); - ***REMOVED*** - - ***REMOVED*** apply the changes - AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId(); - - ***REMOVED*** - - public Algorithm extractAlgorithm(String url) throws IOException ***REMOVED*** - return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** - - -@Override -public void getLogId(Algorithm algo, String vre) ***REMOVED*** - ***REMOVED*** TODO Auto-generated method stub - -***REMOVED*** - - -@Override -public String getLogById(String logId) throws IOException ***REMOVED*** - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** - - - - - -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java deleted file mode 100644 index 8c8e71b..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.util; - -***REMOVED*** -import java.net.Authenticator; -import java.net.InetSocketAddress; -import java.net.PasswordAuthentication; -import java.net.Proxy; -import java.net.ProxySelector; -import java.net.SocketAddress; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -***REMOVED*** - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy; - -interface NetworkConfiguration ***REMOVED*** - - public String getProxyHost(); - - public String getProxyPort(); - - public String getProxyUser(); - - public String getProxyPassword(); - - public String getNonProxyHosts(); - -***REMOVED*** - -class FileBasedProxyConfiguration implements NetworkConfiguration ***REMOVED*** - - private static PropertiesConfiguration configuration; - - public FileBasedProxyConfiguration(String path) ***REMOVED*** - try ***REMOVED*** - ***REMOVED*** load the configuration - configuration = new PropertiesConfiguration(path); - ***REMOVED*** set the reloading strategy to enable hot-configuration - FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy(); - configuration.setReloadingStrategy(fcrs); - ***REMOVED*** catch (ConfigurationException e) ***REMOVED*** - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** - - @Override - public String getProxyHost() ***REMOVED*** - return configuration.getString("proxyHost"); - ***REMOVED*** - - @Override - public String getProxyPort() ***REMOVED*** - return configuration.getString("proxyPort"); - ***REMOVED*** - - @Override - public String getProxyUser() ***REMOVED*** - return configuration.getString("proxyUser"); - ***REMOVED*** - - @Override - public String getProxyPassword() ***REMOVED*** - return configuration.getString("proxyPassword"); - ***REMOVED*** - - @Override - public String getNonProxyHosts() ***REMOVED*** - return configuration.getString("nonProxyHosts"); - ***REMOVED*** - -***REMOVED*** - -public class PropertiesBasedProxySelector extends ProxySelector ***REMOVED*** - - List proxies = null; - - List nonProxyHosts = null; - - public PropertiesBasedProxySelector(String proxySettingsPath) ***REMOVED*** - this(new FileBasedProxyConfiguration(proxySettingsPath)); - ***REMOVED*** - - public PropertiesBasedProxySelector(NetworkConfiguration config) ***REMOVED*** - if (config == null || config.getProxyHost() == null) ***REMOVED*** - this.proxies = null; - return; - ***REMOVED*** - - String host = config.getProxyHost(); - - int port = 80; - - if (config.getProxyPort() != null) ***REMOVED*** - port = Integer.valueOf(config.getProxyPort()); - ***REMOVED*** - - if (config.getNonProxyHosts() != null) ***REMOVED*** - this.nonProxyHosts = Arrays - .asList(config.getNonProxyHosts().split("\\|")); - ***REMOVED*** - - this.proxies = new ArrayList(); - this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host, - port))); - - if (config.getProxyUser() != null) ***REMOVED*** - final String username = config.getProxyUser(); - final String password = config.getProxyPassword(); - - Authenticator.setDefault(new Authenticator() ***REMOVED*** - @Override - protected PasswordAuthentication getPasswordAuthentication() ***REMOVED*** - return new PasswordAuthentication(username, password.toCharArray()); - ***REMOVED*** - ***REMOVED***); - - ***REMOVED*** - ***REMOVED*** - - @Override - public List select(URI uri) ***REMOVED*** - if (this.nonProxyHosts == null) ***REMOVED*** - return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** else ***REMOVED*** - for (String entry : this.nonProxyHosts) ***REMOVED*** - entry = entry.trim(); - if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) ***REMOVED*** - return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** - if (uri.getHost().equals(entry)) ***REMOVED*** - return Arrays.asList(Proxy.NO_PROXY); - ***REMOVED*** - ***REMOVED*** - return this.proxies; - ***REMOVED*** - ***REMOVED*** - - @Override - public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) ***REMOVED*** - - ***REMOVED*** -***REMOVED*** diff --git a/src/main/webapp/WEB-INF/gcube-app.xml b/src/main/webapp/WEB-INF/gcube-app.xml deleted file mode 100644 index 15f231c..0000000 --- a/src/main/webapp/WEB-INF/gcube-app.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - dataminer-pool-manager - dataanalysis - 0.0.1 - - - - - - - - \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/gcube-handlers.xml b/src/main/webapp/WEB-INF/gcube-handlers.xml deleted file mode 100644 index c294252..0000000 --- a/src/main/webapp/WEB-INF/gcube-handlers.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml deleted file mode 100644 index 2a231cb..0000000 --- a/src/main/webapp/WEB-INF/web.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - Archetype Created Web Application - - - - REST-API - org.glassfish.jersey.servlet.ServletContainer - - jersey.config.server.provider.packages - org.gcube.dataanalysis.dataminer.poolmanager.rest - - 1 - - - - REST-API - /rest/* - - - diff --git a/src/test/java/AlgorithmPackageParserTest.java b/src/test/java/AlgorithmPackageParserTest.java deleted file mode 100644 index c728ac2..0000000 --- a/src/test/java/AlgorithmPackageParserTest.java +++ /dev/null @@ -1,117 +0,0 @@ - - -***REMOVED*** -import java.io.InputStream; -***REMOVED*** -***REMOVED*** -import java.util.Vector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; - -public class AlgorithmPackageParserTest ***REMOVED*** - - private static int BUFFER_SIZE = 2048; - - public void extractAllAlgorithms() throws IOException ***REMOVED*** - String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; - List commands = this.extractAddAlgorithmCommands(url); - AlgorithmSet algorithms = new AlgorithmSet(); - for (String cmd : commands) ***REMOVED*** - System.out.println("-----------------------------------------"); - System.out.println(cmd); - AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - System.out.println(aac); - - ***REMOVED*** start creating the algo from the command - Algorithm algo = new Algorithm(); - algo.setAlgorithmType(aac.getAlgorithmType()); - algo.setCategory(aac.getCategory()); - algo.setClazz(aac.getClazz()); - algo.setDescription(aac.getDescription()); - algo.setName(aac.getName()); - algo.setPackageURL(aac.getUrl()); - algo.setSkipJava(aac.getSkipjava()); - - ***REMOVED*** then override with info from the package - if (aac.getUrl().length() > 4) ***REMOVED*** - Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); - if (packagedAlgo != null) ***REMOVED*** - algo.setDependencies(packagedAlgo.getDependencies()); - ***REMOVED*** - ***REMOVED*** - algorithms.addAlgorithm(algo); - break; - ***REMOVED*** - ***REMOVED***to uncomment - ***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - ***REMOVED*** "/gcube/devNext/NextNext"); - ***REMOVED*** - - /** - * Extract 'addAlgorithm' commands from a file containing wiki-table-style - * entries for algorithm. - * - * @return - * @throws IOException - */ - private List extractAddAlgorithmCommands(String listUrl) - throws IOException ***REMOVED*** - URL url = new URL(listUrl); - InputStream is = url.openStream(); - - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = is.read(buffer)) != -1) ***REMOVED*** - s.append(new String(buffer, 0, read)); - ***REMOVED*** - List out = new Vector<>(); - String[] lines = s.toString().split("\n"); - for (String line : lines) ***REMOVED*** - System.out.println("--------------------"); - if (!line.isEmpty()) ***REMOVED*** - String[] parts = line.split("\\|"); - int c = 1; - for (String part : parts) ***REMOVED*** - if (part == null || part.trim().isEmpty()) ***REMOVED*** - continue; - ***REMOVED*** - System.out.println(c + ". " + part); - c++; - if (part.contains("addAlgorithm.sh")) ***REMOVED*** - String cmd = part.trim(); - cmd = cmd.replaceAll("", ""); - cmd = cmd.replaceAll("", ""); - System.out.println(cmd); - ***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - ***REMOVED*** System.out.println(aac); - out.add(cmd); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - /** - * Create an Algorithm starting from the algorithm jar. - * - * @param url - * @return - * @throws IOException - */ - private Algorithm extractAlgorithm(String url) throws IOException ***REMOVED*** - return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** - - public static void main(String[] args) throws Exception ***REMOVED*** -***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector( -***REMOVED*** "/home/ngalante/.proxy-settings")); - - new AlgorithmPackageParserTest().extractAllAlgorithms(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/AnsibleWorkerTest.java b/src/test/java/AnsibleWorkerTest.java deleted file mode 100644 index c7e49a7..0000000 --- a/src/test/java/AnsibleWorkerTest.java +++ /dev/null @@ -1,33 +0,0 @@ - - -import java.io.File; -***REMOVED*** -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleWorkerTest ***REMOVED*** - - public static void main(String[] args) throws IOException ***REMOVED*** - AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString())); - - System.out.println("created worker named " + worker.getWorkerId()); - - worker.setInventory(new Inventory()); - worker.setPlaybook(new Playbook()); - - Role r = new Role(); - r.setName("latex"); - - RoleFile tf = new RoleFile("main", "do something special for " + r.getName()); - r.addTaskFile(tf); - worker.addRole(r); - - worker.apply(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/DataminerPoolManagerTest.java b/src/test/java/DataminerPoolManagerTest.java deleted file mode 100644 index f37288b..0000000 --- a/src/test/java/DataminerPoolManagerTest.java +++ /dev/null @@ -1,227 +0,0 @@ - - -import java.net.ProxySelector; -import java.util.UUID; - -***REMOVED*** -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class DataminerPoolManagerTest ***REMOVED*** - - /* - private static Map domains = new HashMap<>(); - private static Map clusters = new HashMap<>(); - - private static Dependency createDependency(String depName) ***REMOVED*** - String[] parts = depName.split(":"); - Dependency out = new Dependency(); - if(parts.length>1) ***REMOVED*** - out.setType(parts[0]); - out.setName(parts[1]); - ***REMOVED*** else ***REMOVED*** - out.setType("os"); - out.setName(depName); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Algorithm createAlgorithm(String name, String ... deps) ***REMOVED*** - Algorithm a = new Algorithm(); - a.setName(name); - for(String dep:deps) ***REMOVED*** - a.addDependency(createDependency(dep)); - ***REMOVED*** - return a; - ***REMOVED*** - - private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) ***REMOVED*** - AlgorithmSet out = new AlgorithmSet(); - out.setName(name); - for(Algorithm a:algs) ***REMOVED*** - out.addAlgorithm(a); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Domain getDomain(String name) ***REMOVED*** - if(domains.get(name)==null) ***REMOVED*** - Domain d = new Domain(); - d.setName(name); - domains.put(name, d); - return d; - ***REMOVED*** else ***REMOVED*** - return domains.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Host createHost(String hostname, String domainName) ***REMOVED*** - Host out = new Host(); - out.setName(hostname); - Domain d = getDomain(domainName); - out.setDomain(d); - return out; - ***REMOVED*** - - private static Cluster getCluster(String name) ***REMOVED*** - if(clusters.get(name)==null) ***REMOVED*** - Cluster d = new Cluster(); - d.setName(name); - clusters.put(name, d); - return d; - ***REMOVED*** else ***REMOVED*** - return clusters.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Collection extractDependencies() ***REMOVED*** - Collection out = new TreeSet<>(new DependencyComparator()); - for(Cluster c:clusters.values()) ***REMOVED*** - for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED*** - for(Algorithm a:as.getAlgorithms()) ***REMOVED*** - for(Dependency d:a.getDependencies()) ***REMOVED*** - out.add(d); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - private static Collection extractAlgorithms() ***REMOVED*** - Collection out = new TreeSet<>(new AlgorithmComparator()); - for(Cluster c:clusters.values()) ***REMOVED*** - for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED*** - for(Algorithm a:as.getAlgorithms()) ***REMOVED*** - out.add(a); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - static ***REMOVED*** - - Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package"); - Algorithm ensemble = createAlgorithm("ensemble", "python"); - Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah"); - - AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe); - AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble); - AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe); - - Cluster cluster1 = getCluster("cluster-1"); - cluster1.addHost(createHost("host1", "domain1")); - cluster1.addHost(createHost("host2", "domain1")); - cluster1.addHost(createHost("host3", "domain1")); - cluster1.addHost(createHost("host1", "domain2")); - cluster1.addHost(createHost("host2", "domain2")); - - Cluster cluster2 = getCluster("cluster-2"); - cluster2.addHost(createHost("host4", "domain1")); - cluster2.addHost(createHost("host5", "domain1")); - cluster2.addHost(createHost("host6", "domain1")); - cluster2.addHost(createHost("host3", "domain2")); - cluster2.addHost(createHost("host4", "domain2")); - cluster2.addHost(createHost("host5", "domain2")); - - cluster1.addAlgorithmSet(as1); - cluster1.addAlgorithmSet(as2); - - cluster2.addAlgorithmSet(as1); - cluster2.addAlgorithmSet(as3); - - ***REMOVED*** - */ - - public static void main(String[] args) throws Exception ***REMOVED*** - /* - AnsibleBridge ab = new AnsibleBridge(); - ab.printInventoryByDomainAndSets(clusters.values()); - System.out.println("-----------"); - ab.printInventoryBySets(clusters.values()); - - AnsibleWorker worker = ab.createWorker(); - - for(Algorithm a:extractAlgorithms()) ***REMOVED*** - for(Role r:ab.generateRoles(a)) ***REMOVED*** - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - - for(Dependency d:extractDependencies()) ***REMOVED*** - for(Role r:ab.generateRoles(d)) ***REMOVED*** - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - */ - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm algorithm = new Algorithm(); -***REMOVED*** algorithm.setName("ichtyop"); -***REMOVED*** algorithm.setClazz("org.gcube..."); -***REMOVED*** algorithm.setDescription("some description"); -***REMOVED*** -***REMOVED*** Dependency d = new Dependency(); -***REMOVED*** d.setName("libpng"); -***REMOVED*** d.setType("os"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-other-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm ewe = new Algorithm(); -***REMOVED*** ewe.setName("ewe"); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** ewe.addDependency(d); - - AlgorithmSet algorithms = new AlgorithmSet(); - algorithms.setName("dummy-set"); - - -***REMOVED*** algorithms.addAlgorithm(algorithm); -***REMOVED*** algorithms.addAlgorithm(ewe); - - Algorithm ensemble = new Algorithm(); - ensemble.setName("ewe"); - ensemble.setCategory("a"); - ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http:***REMOVED***data.d4science.org/eDd5b2ovMmpSVEZaZWYvY3g2bDZNcGRFNUIxNi85UFlHbWJQNStIS0N6Yz0"); - ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); - ensemble.setSkipJava("N"); - ensemble.setDescription("test"); - - Dependency d = new Dependency(); - d.setName("libpng3"); - d.setType("os"); - ensemble.addDependency(d); - algorithms.addAlgorithm(ensemble); - - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID()); - - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/ISClientTest.java b/src/test/java/ISClientTest.java deleted file mode 100644 index 586b9aa..0000000 --- a/src/test/java/ISClientTest.java +++ /dev/null @@ -1,17 +0,0 @@ - - -import java.net.ProxySelector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class ISClientTest ***REMOVED*** - - public static void main(String[] args) ***REMOVED*** - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(new ISClient().listDataminersInVRE()); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java deleted file mode 100644 index bc92626..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -***REMOVED*** -import java.io.InputStream; -import java.net.ProxySelector; -***REMOVED*** -***REMOVED*** -import java.util.Vector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class AlgorithmPackageParserTest ***REMOVED*** - - private static int BUFFER_SIZE = 2048; - - public void extractAllAlgorithms() throws IOException ***REMOVED*** - String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; - List commands = this.extractAddAlgorithmCommands(url); - AlgorithmSet algorithms = new AlgorithmSet(); - for (String cmd : commands) ***REMOVED*** - System.out.println("-----------------------------------------"); - System.out.println(cmd); - AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - System.out.println(aac); - - ***REMOVED*** start creating the algo from the command - Algorithm algo = new Algorithm(); - algo.setAlgorithmType(aac.getAlgorithmType()); - algo.setCategory(aac.getCategory()); - algo.setClazz(aac.getClazz()); - algo.setDescription(aac.getDescription()); - algo.setName(aac.getName()); - algo.setPackageURL(aac.getUrl()); - algo.setSkipJava(aac.getSkipjava()); - - ***REMOVED*** then override with info from the package - if (aac.getUrl().length() > 4) ***REMOVED*** - Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); - if (packagedAlgo != null) ***REMOVED*** - algo.setDependencies(packagedAlgo.getDependencies()); - ***REMOVED*** - ***REMOVED*** - algorithms.addAlgorithm(algo); - break; - ***REMOVED*** - ***REMOVED***to uncomment - ***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - ***REMOVED*** "/gcube/devNext/NextNext"); - ***REMOVED*** - - /** - * Extract 'addAlgorithm' commands from a file containing wiki-table-style - * entries for algorithm. - * - * @return - * @throws IOException - */ - private List extractAddAlgorithmCommands(String listUrl) - throws IOException ***REMOVED*** - URL url = new URL(listUrl); - InputStream is = url.openStream(); - - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = is.read(buffer)) != -1) ***REMOVED*** - s.append(new String(buffer, 0, read)); - ***REMOVED*** - List out = new Vector<>(); - String[] lines = s.toString().split("\n"); - for (String line : lines) ***REMOVED*** - System.out.println("--------------------"); - if (!line.isEmpty()) ***REMOVED*** - String[] parts = line.split("\\|"); - int c = 1; - for (String part : parts) ***REMOVED*** - if (part == null || part.trim().isEmpty()) ***REMOVED*** - continue; - ***REMOVED*** - System.out.println(c + ". " + part); - c++; - if (part.contains("addAlgorithm.sh")) ***REMOVED*** - String cmd = part.trim(); - cmd = cmd.replaceAll("", ""); - cmd = cmd.replaceAll("", ""); - System.out.println(cmd); - ***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - ***REMOVED*** System.out.println(aac); - out.add(cmd); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - /** - * Create an Algorithm starting from the algorithm jar. - * - * @param url - * @return - * @throws IOException - */ - private Algorithm extractAlgorithm(String url) throws IOException ***REMOVED*** - return new AlgorithmPackageParser().parsePackage(url); - ***REMOVED*** - - public static void main(String[] args) throws Exception ***REMOVED*** -***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector( -***REMOVED*** "/home/ngalante/.proxy-settings")); - - new AlgorithmPackageParserTest().extractAllAlgorithms(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java deleted file mode 100644 index 7f6cc7e..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.io.File; -***REMOVED*** -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleWorkerTest ***REMOVED*** - - public static void main(String[] args) throws IOException ***REMOVED*** - AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString())); - - System.out.println("created worker named " + worker.getWorkerId()); - - worker.setInventory(new Inventory()); - worker.setPlaybook(new Playbook()); - - Role r = new Role(); - r.setName("latex"); - - RoleFile tf = new RoleFile("main", "do something special for " + r.getName()); - r.addTaskFile(tf); - worker.addRole(r); - - worker.apply(); - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java deleted file mode 100644 index ca90267..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java +++ /dev/null @@ -1,227 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.net.ProxySelector; -import java.util.UUID; - -***REMOVED*** -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class DataminerPoolManagerTest ***REMOVED*** - - /* - private static Map domains = new HashMap<>(); - private static Map clusters = new HashMap<>(); - - private static Dependency createDependency(String depName) ***REMOVED*** - String[] parts = depName.split(":"); - Dependency out = new Dependency(); - if(parts.length>1) ***REMOVED*** - out.setType(parts[0]); - out.setName(parts[1]); - ***REMOVED*** else ***REMOVED*** - out.setType("os"); - out.setName(depName); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Algorithm createAlgorithm(String name, String ... deps) ***REMOVED*** - Algorithm a = new Algorithm(); - a.setName(name); - for(String dep:deps) ***REMOVED*** - a.addDependency(createDependency(dep)); - ***REMOVED*** - return a; - ***REMOVED*** - - private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) ***REMOVED*** - AlgorithmSet out = new AlgorithmSet(); - out.setName(name); - for(Algorithm a:algs) ***REMOVED*** - out.addAlgorithm(a); - ***REMOVED*** - return out; - ***REMOVED*** - - private static Domain getDomain(String name) ***REMOVED*** - if(domains.get(name)==null) ***REMOVED*** - Domain d = new Domain(); - d.setName(name); - domains.put(name, d); - return d; - ***REMOVED*** else ***REMOVED*** - return domains.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Host createHost(String hostname, String domainName) ***REMOVED*** - Host out = new Host(); - out.setName(hostname); - Domain d = getDomain(domainName); - out.setDomain(d); - return out; - ***REMOVED*** - - private static Cluster getCluster(String name) ***REMOVED*** - if(clusters.get(name)==null) ***REMOVED*** - Cluster d = new Cluster(); - d.setName(name); - clusters.put(name, d); - return d; - ***REMOVED*** else ***REMOVED*** - return clusters.get(name); - ***REMOVED*** - ***REMOVED*** - - private static Collection extractDependencies() ***REMOVED*** - Collection out = new TreeSet<>(new DependencyComparator()); - for(Cluster c:clusters.values()) ***REMOVED*** - for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED*** - for(Algorithm a:as.getAlgorithms()) ***REMOVED*** - for(Dependency d:a.getDependencies()) ***REMOVED*** - out.add(d); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - private static Collection extractAlgorithms() ***REMOVED*** - Collection out = new TreeSet<>(new AlgorithmComparator()); - for(Cluster c:clusters.values()) ***REMOVED*** - for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED*** - for(Algorithm a:as.getAlgorithms()) ***REMOVED*** - out.add(a); - ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return out; - ***REMOVED*** - - static ***REMOVED*** - - Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package"); - Algorithm ensemble = createAlgorithm("ensemble", "python"); - Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah"); - - AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe); - AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble); - AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe); - - Cluster cluster1 = getCluster("cluster-1"); - cluster1.addHost(createHost("host1", "domain1")); - cluster1.addHost(createHost("host2", "domain1")); - cluster1.addHost(createHost("host3", "domain1")); - cluster1.addHost(createHost("host1", "domain2")); - cluster1.addHost(createHost("host2", "domain2")); - - Cluster cluster2 = getCluster("cluster-2"); - cluster2.addHost(createHost("host4", "domain1")); - cluster2.addHost(createHost("host5", "domain1")); - cluster2.addHost(createHost("host6", "domain1")); - cluster2.addHost(createHost("host3", "domain2")); - cluster2.addHost(createHost("host4", "domain2")); - cluster2.addHost(createHost("host5", "domain2")); - - cluster1.addAlgorithmSet(as1); - cluster1.addAlgorithmSet(as2); - - cluster2.addAlgorithmSet(as1); - cluster2.addAlgorithmSet(as3); - - ***REMOVED*** - */ - - public static void main(String[] args) throws Exception ***REMOVED*** - /* - AnsibleBridge ab = new AnsibleBridge(); - ab.printInventoryByDomainAndSets(clusters.values()); - System.out.println("-----------"); - ab.printInventoryBySets(clusters.values()); - - AnsibleWorker worker = ab.createWorker(); - - for(Algorithm a:extractAlgorithms()) ***REMOVED*** - for(Role r:ab.generateRoles(a)) ***REMOVED*** - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - - for(Dependency d:extractDependencies()) ***REMOVED*** - for(Role r:ab.generateRoles(d)) ***REMOVED*** - worker.addRole(r); - ***REMOVED*** - ***REMOVED*** - */ - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm algorithm = new Algorithm(); -***REMOVED*** algorithm.setName("ichtyop"); -***REMOVED*** algorithm.setClazz("org.gcube..."); -***REMOVED*** algorithm.setDescription("some description"); -***REMOVED*** -***REMOVED*** Dependency d = new Dependency(); -***REMOVED*** d.setName("libpng"); -***REMOVED*** d.setType("os"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("some-other-r-package"); -***REMOVED*** d.setType("cran"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** algorithm.addDependency(d); -***REMOVED*** -***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc) -***REMOVED*** Algorithm ewe = new Algorithm(); -***REMOVED*** ewe.setName("ewe"); -***REMOVED*** -***REMOVED*** d = new Dependency(); -***REMOVED*** d.setName("voodoo"); -***REMOVED*** d.setType("custom"); -***REMOVED*** ewe.addDependency(d); - - AlgorithmSet algorithms = new AlgorithmSet(); - algorithms.setName("dummy-set"); - - -***REMOVED*** algorithms.addAlgorithm(algorithm); -***REMOVED*** algorithms.addAlgorithm(ewe); - - Algorithm ensemble = new Algorithm(); - ensemble.setName("ensemble"); - ensemble.setCategory("ICHTHYOP_MODEL"); - ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http:***REMOVED***data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0"); - ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); - ensemble.setSkipJava("N"); - ensemble.setDescription("test"); - - Dependency d = new Dependency(); - d.setName("libpng3"); - d.setType("os"); - ensemble.addDependency(d); - algorithms.addAlgorithm(ensemble); - - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID()); - - ***REMOVED*** - -***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java deleted file mode 100644 index d4c5dbc..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.net.ProxySelector; - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class ISClientTest ***REMOVED*** - - public static void main(String[] args) ***REMOVED*** - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(new ISClient().listDataminersInVRE()); - ***REMOVED*** - -***REMOVED***