From ad7006ef2d158d6fdd473858698dc343dc6361f1 Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Fri, 3 Mar 2017 18:07:54 +0000 Subject: [PATCH] Share project "dataminer-pool-manager" into "http://svn.research-infrastructures.eu/d4science/gcube" git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@144668 82a268e6-3cf1-43bd-a215-b396298e98cf --- .classpath | 34 -- .project | 42 --- distro/LICENSE | 1 - distro/README | 60 ---- distro/descriptor.xml | 36 --- distro/profile.xml | 25 -- pom.xml | 125 -------- .../poolmanager/ansible/AnsibleWorker.java | 135 -------- .../ansible/model/AnsibleHost.java | 19 -- .../poolmanager/ansible/model/HostGroup.java | 29 -- .../poolmanager/ansible/model/Inventory.java | 37 --- .../poolmanager/ansible/model/Playbook.java | 50 --- .../poolmanager/ansible/model/Role.java | 51 --- .../poolmanager/ansible/model/RoleFile.java | 54 ---- .../ansiblebridge/AnsibleBridge.java | 276 ----------------- .../ansiblebridge/AnsibleSerializeHelper.java | 119 ------- .../template/AlgorithmPackage.java | 71 ----- .../template/CranDependencyPackage.java | 11 - .../template/CustomDependencyPackage.java | 65 ---- .../template/CustomRoleManager.java | 32 -- .../template/DependencyPackage.java | 55 ---- .../template/OSDependencyPackage.java | 11 - .../template/StaticRoleManager.java | 37 --- .../template/TemplateManager.java | 95 ------ .../poolmanager/clients/ISClient.java | 66 ---- .../poolmanager/datamodel/Action.java | 35 --- .../poolmanager/datamodel/Algorithm.java | 118 ------- .../poolmanager/datamodel/AlgorithmSet.java | 49 --- .../poolmanager/datamodel/Cluster.java | 73 ----- .../poolmanager/datamodel/Dependency.java | 29 -- .../poolmanager/datamodel/Domain.java | 15 - .../dataminer/poolmanager/datamodel/Host.java | 39 --- .../comparator/AlgorithmComparator.java | 15 - .../comparator/DependencyComparator.java | 18 -- .../datamodel/comparator/HostComparator.java | 17 - .../process/AddAlgorithmCommand.java | 103 ------- .../process/AlgorithmPackageParser.java | 290 ------------------ .../poolmanager/rest/PoolManager.java | 24 -- .../poolmanager/rest/RestPoolManager.java | 144 --------- .../service/DataminerPoolManager.java | 262 ---------------- .../util/PropertiesBasedProxySelector.java | 145 --------- src/main/webapp/WEB-INF/gcube-app.xml | 13 - src/main/webapp/WEB-INF/gcube-handlers.xml | 11 - src/main/webapp/WEB-INF/web.xml | 24 -- src/test/java/AlgorithmPackageParserTest.java | 117 ------- src/test/java/AnsibleWorkerTest.java | 33 -- src/test/java/DataminerPoolManagerTest.java | 227 -------------- src/test/java/ISClientTest.java | 17 - .../AlgorithmPackageParserTest.java | 119 ------- .../AnsibleWorkerTest.java | 33 -- .../DataminerPoolManagerTest.java | 227 -------------- .../dataminerpoolmanager/ISClientTest.java | 17 - 52 files changed, 3750 deletions(-) delete mode 100644 .classpath delete mode 100644 .project delete mode 100644 distro/LICENSE delete mode 100644 distro/README delete mode 100644 distro/descriptor.xml delete mode 100644 distro/profile.xml delete mode 100644 pom.xml delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java delete mode 100644 src/main/webapp/WEB-INF/gcube-app.xml delete mode 100644 src/main/webapp/WEB-INF/gcube-handlers.xml delete mode 100644 src/main/webapp/WEB-INF/web.xml delete mode 100644 src/test/java/AlgorithmPackageParserTest.java delete mode 100644 src/test/java/AnsibleWorkerTest.java delete mode 100644 src/test/java/DataminerPoolManagerTest.java delete mode 100644 src/test/java/ISClientTest.java delete mode 100644 src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java delete mode 100644 src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java delete mode 100644 src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java delete mode 100644 src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java diff --git a/.classpath b/.classpath deleted file mode 100644 index 70eaec6..0000000 --- a/.classpath +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/.project b/.project deleted file mode 100644 index 367d621..0000000 --- a/.project +++ /dev/null @@ -1,42 +0,0 @@ - - - dataminer-pool-manager - - - - - - org.eclipse.wst.jsdt.core.javascriptValidator - - - - - org.eclipse.wst.common.project.facet.core.builder - - - - - org.eclipse.jdt.core.javabuilder - - - - - org.eclipse.wst.validation.validationbuilder - - - - - org.eclipse.m2e.core.maven2Builder - - - - - - org.eclipse.jem.workbench.JavaEMFNature - org.eclipse.wst.common.modulecore.ModuleCoreNature - org.eclipse.jdt.core.javanature - org.eclipse.m2e.core.maven2Nature - org.eclipse.wst.common.project.facet.core.nature - org.eclipse.wst.jsdt.core.jsNature - - diff --git a/distro/LICENSE b/distro/LICENSE deleted file mode 100644 index 2d9616a..0000000 --- a/distro/LICENSE +++ /dev/null @@ -1 +0,0 @@ -${gcube.license} \ No newline at end of file diff --git a/distro/README b/distro/README deleted file mode 100644 index 1abbf28..0000000 --- a/distro/README +++ /dev/null @@ -1,60 +0,0 @@ -The gCube System - ${name} --------------------------------------------------- - -${description} - -${gcube.description} - -${gcube.funding} - - -Version --------------------------------------------------- -${version} (${buildDate}) - -Please see the file named "changelog.xml" in this directory for the release notes. - - -Authors --------------------------------------------------- -* Paolo Fabriani (paolo.fabriani-AT-eng.it) - Engineering Ingegneria Informatica S.p.A., Italy - - -Maintainers --------------------------------------------------- -* Paolo Fabriani (paolo.fabriani-AT-eng.it) - Engineering Ingegneria Informatica S.p.A., Italy - - -Download information --------------------------------------------------- -Source code is available from SVN: - ${scm.url} - -Binaries can be downloaded from the gCube website: - ${gcube.website} - - -Installation --------------------------------------------------- -Installation documentation is available on-line in the gCube Wiki: - ${gcube.wikiRoot}/[admin guide page name here] - - -Documentation --------------------------------------------------- -Documentation is available on-line in the gCube Wiki: - ${gcube.wikiRoot}[user guide page name here] - ${gcube.wikiRoot}[developer guide page name here] - - -Support --------------------------------------------------- -Bugs and support requests can be reported in the gCube issue tracking tool: - ${gcube.issueTracking} - - -Licensing --------------------------------------------------- -This software is licensed under the terms you may find in the file named "LICENSE" in this directory. \ No newline at end of file diff --git a/distro/descriptor.xml b/distro/descriptor.xml deleted file mode 100644 index 7e10090..0000000 --- a/distro/descriptor.xml +++ /dev/null @@ -1,36 +0,0 @@ - - - servicearchive - - - tar.gz - - - / - - - - ${distroDirectory} - / - true - - README - LICENSE - profile.xml - - 755 - true - - - - - - target/${build.finalName}.war - /${artifactId} - - - - \ No newline at end of file diff --git a/distro/profile.xml b/distro/profile.xml deleted file mode 100644 index e58c7f8..0000000 --- a/distro/profile.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - Library - - ${description} - DataminerPoolManager - ${artifactId} - 1.0.0 - - - ${artifactId} - ${version} - - ${groupId} - ${artifactId} - ${version} - - - ${build.finalName}.jar - - - - - \ No newline at end of file diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 37fe797..0000000 --- a/pom.xml +++ /dev/null @@ -1,125 +0,0 @@ - - 4.0.0 - - - maven-parent - org.gcube.tools - 1.0.0 - - - - org.gcube.dataanalysis - dataminer-pool-manager - 1.0.0-SNAPSHOT - war - - dataminer-pool-manager - - - - - distro - UTF-8 - - - - - commons-io - commons-io - [2.5.0,2.6.0) - - - org.antlr - stringtemplate - [4.0.0, 4.1.0) - - - commons-configuration - commons-configuration - 1.10 - - - org.gcube.resources.discovery - ic-client - provided - - - org.yaml - snakeyaml - 1.16 - - - org.glassfish.jersey.media - jersey-media-json-jackson - 2.23.1 - - - org.glassfish.jersey.core - jersey-client - 2.22.1 - - - - org.glassfish.jersey.containers - jersey-container-servlet - 2.22.1 - - - - org.slf4j - slf4j-api - provided - - - - - - - - - - org.gcube.distribution - maven-smartgears-bom - LATEST - pom - import - - - - - - - - - maven-compiler-plugin - - 1.7 - 1.7 - - - - - org.apache.maven.plugins - maven-assembly-plugin - 2.2 - - - ${distroDirectory}/descriptor.xml - - - - - servicearchive - install - - single - - - - - - - - - diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java deleted file mode 100644 index 0a556bf..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ /dev/null @@ -1,135 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.PrintStream; -import java.util.Scanner; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - -/** - * This class is responsible for the interface with ansible, retrieving log, - * etc. etc. It's not supposed to access templates and static stuff files. It - * does not know the service datamodel. - * - * @author paolo - * - */ -public class AnsibleWorker { - - /** - * The name of the inventory - */ - private static String INVENTORY_NAME = "inventory.yaml"; - - /** - * The directory containing roles - */ - private static String ROLES_DIR = "roles"; - - /** - * The name of the playbook - */ - private static String PLAYBOOK_NAME = "playbook.yaml"; - - /** - * The root of the worker. This corresponds to a standard ansible working dir. - */ - private File workerRoot; - - public AnsibleWorker(File root) { - this.workerRoot = root; - this.ensureWorkStructure(); - } - - public File getWorkdir() { - return this.workerRoot; - } - - public File getRolesDir() { - return new File(this.getWorkdir(), ROLES_DIR); - } - - public String getWorkerId() { - return this.workerRoot.getName(); - } - - public void ensureWorkStructure() { - // generate root - this.getWorkdir().mkdirs(); - } - - public void removeWorkStructure() { - // remove the working dir - this.getWorkdir().delete(); - } - - public File getPlaybookFile() { - return new File(this.getWorkdir(), PLAYBOOK_NAME); - } - - public File getInventoryFile() { - return new File(this.getWorkdir(), INVENTORY_NAME); - } - - - public void setInventory(Inventory inventory) throws IOException { - // serialize the string to the 'inventory' file - AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile()); - } - - public void setPlaybook(Playbook playbook) throws IOException { - // serialize the string to the 'playbook' file - AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile()); - } - - public void addRole(Role r) throws IOException { - // Serialize role in the workdir - AnsibleSerializeHelper.serializeRole(r, this.getRolesDir()); - } - - - - public void apply() throws IOException { - // TODO execute the playbook and return output - System.out.println(this.getWorkdir()); - try { - Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath()); - - inheritIO(p.getInputStream(), System.out); - inheritIO(p.getErrorStream(), System.err); - - } catch (IOException e) { - e.printStackTrace(); - } - //System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName()); - } - - - - private static void inheritIO(final InputStream src, final PrintStream dest) { - new Thread(new Runnable() { - public void run() { - Scanner sc = new Scanner(src); - while (sc.hasNextLine()) { - dest.println(sc.nextLine()); - } - } - }).start(); - } - - /** - * Destroy the worker: - * - remove the working dir - */ - public void destroy() { - this.removeWorkStructure(); - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java deleted file mode 100644 index 9ed34b3..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -public class AnsibleHost { - - private String name; - - public AnsibleHost(String name) { - this.name = name; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java deleted file mode 100644 index 077e1a1..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class HostGroup { - - private String name; - - private Collection hosts; - - public HostGroup(String name) { - this.name = name; - this.hosts = new Vector<>(); - } - - public void addHost(AnsibleHost h) { - this.hosts.add(h); - } - - public String getName() { - return this.name; - } - - public Collection getHosts() { - return new Vector<>(this.hosts); - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java deleted file mode 100644 index 4d6400d..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class Inventory { - - private Collection groups; - - public Inventory() { - this.groups = new Vector<>(); - } - - public void addGroup(HostGroup group) { - this.groups.add(group); - } - - public void addHost(AnsibleHost h, String groupName) { - this.getGroup(groupName).addHost(h); - } - - private HostGroup getGroup(String groupName) { - for (HostGroup hg : this.groups) { - if (groupName.equals(hg.getName())) { - return hg; - } - } - HostGroup hg = new HostGroup(groupName); - this.groups.add(hg); - return hg; - } - - public Collection getHostGroups() { - return new Vector<>(this.groups); - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java deleted file mode 100644 index c08c7e2..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java +++ /dev/null @@ -1,50 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.List; -import java.util.Vector; - -public class Playbook { - - private String hostGroupName; - - private List roles; - - private String remote_user; - - public Playbook() { - this.roles = new Vector<>(); - } - - public void addRole(String role) { - roles.add(role); - } - - public void applyTo(String hostGroupName) { - this.hostGroupName = hostGroupName; - } - - public String getHostGroupName() { - return hostGroupName; - } - - public List getRoles() { - return new Vector<>(roles); - } - -public String getRemote_user() { - return remote_user; -} - -public void setRemote_user(String remote_user) { - this.remote_user = remote_user; -} - -public void setHostGroupName(String hostGroupName) { - this.hostGroupName = hostGroupName; -} - -public void setRoles(List roles) { - this.roles = roles; -} - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java deleted file mode 100644 index 3f3aa33..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java +++ /dev/null @@ -1,51 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -import java.util.Collection; -import java.util.Vector; - -public class Role { - - /** - * The name of the role - */ - private String name; - - private Collection tasks; - - private Collection meta; - - public Role() { - this.tasks = new Vector<>(); - this.meta = new Vector<>(); - } - - public Role(String name) { - this(); - this.name = name; - } - - public void addTaskFile(RoleFile tf) { - this.tasks.add(tf); - } - - public void addMeta(RoleFile tf) { - this.meta.add(tf); - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Collection getTaskFiles() { - return new Vector<>(this.tasks); - } - - public Collection getMeta() { - return new Vector<>(this.meta); - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java deleted file mode 100644 index 3cff918..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java +++ /dev/null @@ -1,54 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model; - -public class RoleFile { - - /** - * The path to the file, starting from the role root - */ - private String path; - - /** - * The name of the task file - */ - private String name; - - /** - * The content of the task file - * @return - */ - private String content; - - public RoleFile() { - } - - public RoleFile(String name, String content) { - this(); - this.setName(name); - this.setContent(content); - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getContent() { - return content; - } - - public void setContent(String content) { - this.content = content; - } - - public String getPath() { - return path; - } - - public void setPath(String path) { - this.path = path; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java deleted file mode 100644 index 8e97a20..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ /dev/null @@ -1,276 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.UUID; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.AlgorithmPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CranDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomRoleManager; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.OSDependencyPackage; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.StaticRoleManager; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; -import org.slf4j.LoggerFactory; - -public class AnsibleBridge { - private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); - - - /** - * The workdir for this service - */ - private String dpmRoot; - - public AnsibleBridge() { - this(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); - } - - public AnsibleBridge(String root) { - this.dpmRoot = root; - this.ensureServiceRoot(); - } - - private void ensureServiceRoot() { - // generate root - new File(dpmRoot).mkdirs(); - // 'template' is for template roles - this.getTemplatesDir().mkdirs(); - // 'static' is for custom roles - this.getCustomDir().mkdirs(); - // 'work' is for temporary working directories - this.getWorkDir().mkdirs(); - } - - private File getWorkDir() { - return new File(this.dpmRoot, "work"); - } - - private File getTemplatesDir() { - return new File(this.dpmRoot, "templates"); - } - - private File getCustomDir() { - return new File(this.dpmRoot, "custom"); - } - - public AnsibleWorker createWorker() { - File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); - AnsibleWorker worker = new AnsibleWorker(workerRoot); - return worker; - } - - /** - * Groups hosts by domain and algorithm sets - * @param clusters - */ - public void printInventoryByDomainAndSets(Collection clusters) { - Map> inventory = new TreeMap<>(); - for(Cluster cluster:clusters) { - for(AlgorithmSet as:cluster.getAlgorithmSets()) { - String asName = as.getName(); - for(Host h:cluster.getHosts()) { - String domain = h.getDomain().getName(); - String key = String.format("[%s@%s]", asName, domain); - Set hosts = inventory.get(key); - if(hosts==null) { - hosts = new TreeSet<>(new HostComparator()); - inventory.put(key, hosts); - } - hosts.add(h); - } - - } - } - for(String key:inventory.keySet()) { - System.out.println(key); - Collection hosts = inventory.get(key); - for(Host h:hosts) { - System.out.println(h.getName()+"."+h.getDomain().getName()); - } - System.out.println(); - } - } - - /** - * Groups hosts by algorithm sets only - * @param clusters - */ - public void printInventoryBySets(Collection clusters) { - Map> inventory = new TreeMap<>(); - for (Cluster cluster : clusters) { - for (AlgorithmSet as : cluster.getAlgorithmSets()) { - String asName = as.getName(); - for (Host h : cluster.getHosts()) { - String key = String.format("[%s]", asName); - Set hosts = inventory.get(key); - if (hosts == null) { - hosts = new TreeSet<>(new HostComparator()); - inventory.put(key, hosts); - } - hosts.add(h); - } - - } - } - for (String key : inventory.keySet()) { - System.out.println(key); - Collection hosts = inventory.get(key); - for (Host h : hosts) { - System.out.println(h.getName()+"."+h.getDomain().getName()); - } - System.out.println(); - } - } - - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException { - - - return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString()); - } - - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid) throws IOException { - AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); - - - List algoRoles = new Vector<>(); - - // add algorithms and dependencies to the worker - for (Algorithm a : as.getAlgorithms()) { - for (Role r : this.generateRoles(a)) { - algoRoles.add(r); - worker.addRole(r); - } - for (Dependency d : a.getDependencies()) { - for (Role r : this.generateRoles(d)) { - worker.addRole(r); - } - } - } - - // add static roles - for(Role r:this.getStaticRoleManager().getStaticRoles()) { - worker.addRole(r); - } - - // generate the inventory - Inventory inventory = new Inventory(); - for (Host h : cluster.getHosts()) { - AnsibleHost ah = new AnsibleHost(h.getName()); - inventory.addHost(ah, "universe"); - inventory.addHost(ah, "d4science"); - } - worker.setInventory(inventory); - - // generate the playbook - Playbook playbook = new Playbook(); - playbook.setRemote_user("root"); - playbook.applyTo("universe"); - for(Role r:algoRoles) { - // add only 'add' roles - if(!r.getName().endsWith("remove")) { - playbook.addRole(r.getName()); - } - } - - worker.setPlaybook(playbook); - - // execute and save log locally - PrintStream console = System.out; - File path = new File(worker.getWorkdir() + File.separator + "logs"); - path.mkdirs(); - File n = new File(path + File.separator + worker.getWorkerId()); - FileOutputStream fos = new FileOutputStream(n); - PrintStream ps = new PrintStream(fos); - System.setOut(ps); - System.setErr(ps); - //System.setErr(console); - - worker.apply(); - //System.setOut(console); - //worker.apply(); - System.out.println("Log stored to to " + n.getAbsolutePath()); - - // destroy the worker - worker.destroy(); - return worker; - } - - - - private TemplateManager getTemplateManager() { - return new TemplateManager(this.dpmRoot+"/templates"); - } - - private CustomRoleManager getCustomRoleManager() { - return new CustomRoleManager(this.dpmRoot+"/custom"); - } - - private StaticRoleManager getStaticRoleManager() { - return new StaticRoleManager(this.dpmRoot+"/static"); - } - - /** - * Generate all roles for this dependency - * @param d - */ - public Collection generateRoles(Dependency d) { - Collection roles = new Vector<>(); - - - if("os".equalsIgnoreCase(d.getType())) { - OSDependencyPackage pkg = new OSDependencyPackage(d); - if(pkg!=null) { - roles.addAll(pkg.getRoles(this.getTemplateManager())); - } - - } else if("custom".equalsIgnoreCase(d.getType())) { - CustomDependencyPackage pkg = new CustomDependencyPackage(d); - if(pkg!=null) { - roles.addAll(pkg.getRoles(this.getCustomRoleManager())); - } - } - - else if("github".equalsIgnoreCase(d.getType())) { - CranDependencyPackage pkg = new CranDependencyPackage(d); - if(pkg!=null) { - roles.addAll(pkg.getRoles(this.getTemplateManager())); - } - } - else if("cran".equalsIgnoreCase(d.getType())) { - CranDependencyPackage pkg = new CranDependencyPackage(d); - if(pkg!=null) { - roles.addAll(pkg.getRoles(this.getTemplateManager())); - } - - - } - return roles; - } - - public Collection generateRoles(Algorithm a) { - AlgorithmPackage pkg = new AlgorithmPackage(a); - return pkg.getRoles(this.getTemplateManager()); - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java deleted file mode 100644 index fa0496c..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.PrintWriter; - -import org.apache.commons.io.IOUtils; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.HostGroup; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleSerializeHelper { - - public static void serialize(Inventory inventory, File inventoryFile) throws IOException { - String out = ""; - for(HostGroup hg:inventory.getHostGroups()) { - out+=String.format("[%s]\n", hg.getName()); - for(AnsibleHost h:hg.getHosts()) { - out+=h.getName()+"\n"; - } - out+="\n"; - } - out = out.trim(); - serialize(out, inventoryFile); - } - - public static void serialize(Playbook playbook, File playbookFile) throws IOException { - String out = "- hosts: " + playbook.getHostGroupName() + "\n"; - out += " remote_user: "+playbook.getRemote_user()+"\n"; - out+=" roles:\n"; - for(String r:playbook.getRoles()) { - out+=" - " + r+"\n"; - } - out+=" vars:\n"; - out+=" os_package_state: present\n"; - out = out.trim(); - serialize(out, playbookFile); - } - - public static void serializeRole(Role r, File dir) throws IOException { - // create root - File root = new File(dir, r.getName()); - root.mkdirs(); - - // create tasks - if(r.getTaskFiles().size()>0) { - File tasks = new File(root, "tasks"); - tasks.mkdirs(); - for(RoleFile tf: r.getTaskFiles()) { - serializeTask(tf, tasks); - } - } - - // create meta - if(r.getMeta().size()>0) { - File meta = new File(root, "meta"); - meta.mkdirs(); - for(RoleFile tf: r.getMeta()) { - serializeTask(tf, meta); - } - } - } - - public static void serializeTask(RoleFile tf, File dir) throws IOException { - File f = new File(dir, tf.getName()); - serialize(tf.getContent().trim(), f); - } - - public static void serialize(String s, File f) throws IOException { - PrintWriter out = new PrintWriter(f); - out.println(s); - out.close(); - } - - public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException { - Role out = new Role(); - out.setName(roleDir.getName()); - - if(!roleDir.exists()) { - throw new FileNotFoundException(); - } - - try { - File tasksDir = new File(roleDir, "tasks"); - if(tasksDir.exists()) { - for(File main:tasksDir.listFiles()) { - String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); - RoleFile tf = new RoleFile(main.getName(), content); - tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); - out.addTaskFile(tf); - } - } - } catch(FileNotFoundException e) { - e.printStackTrace(); - } - - try { - File metaDir = new File(roleDir, "meta"); - if(metaDir.exists()) { - for(File main:metaDir.listFiles()) { - String content = IOUtils.toString(new FileInputStream(main), "UTF-8"); - RoleFile tf = new RoleFile(main.getName(), content); - tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1)); - out.addMeta(tf); - } - } - } catch(FileNotFoundException e) { - e.printStackTrace(); - } - - return out; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java deleted file mode 100644 index 9393c8e..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Vector; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class AlgorithmPackage { - - private Algorithm algorithm; - - public AlgorithmPackage(Algorithm a) { - this.algorithm = a; - } - - protected Map getDictionary(Algorithm a) { - Map out = new HashMap(); - out.put("name", a.getName()); - out.put("category", a.getCategory()); - out.put("class", a.getClazz()); - out.put("atype", a.getAlgorithmType()); - out.put("skipjava", a.getSkipJava()); - out.put("vre", ScopeProvider.instance.get()); - //out.put("vre", "FAKE_VRE"); - out.put("packageurl", a.getPackageURL()); - out.put("description", a.getDescription()); - String deps = ""; - - for(Dependency d:a.getDependencies()) { - deps+=String.format("- { role: %s }\n", d.getType()+"-"+d.getName().replaceAll("/", "-")); - } - deps = deps.trim(); - out.put("dependencies", deps); - return out; - } - - protected Algorithm getAlgorithm() { - return this.algorithm; - } - - public Collection getRoles(TemplateManager tm) { - Collection out = new Vector<>(); - for(String mode:new String[]{"add"}) { // "remove", "update" - String roleName = "gcube-algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode); - try { - // find template - Role template = tm.getRoleTemplate("gcube-algorithm-" + mode); - // - if(template!=null) { - Map dictionary = this.getDictionary(this.getAlgorithm()); - Role r = tm.fillRoleTemplate(template, dictionary); - r.setName(roleName); - out.add(r); - } else { - System.out.println("WARNING: template is null"); - } - } catch (NoSuchElementException e) { -// e.printStackTrace(); - System.out.println("WARNING: no template found for " + roleName); - } - } - return out; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java deleted file mode 100644 index baa4910..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class CranDependencyPackage extends DependencyPackage { - - public CranDependencyPackage(Dependency d) { - super(d); - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java deleted file mode 100644 index a00d08a..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java +++ /dev/null @@ -1,65 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.NoSuchElementException; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class CustomDependencyPackage extends DependencyPackage { - - public CustomDependencyPackage(Dependency d) { - super(d); - } - - private String getCustomRepositoryLocation(String ansibleRoot) { - return ansibleRoot+"/custom"; - } - - /* - public void serializeTo(String ansibleRoot) { - for(String mode:new String[]{"add", "remove", "update"}) { - // look for roles in the 'custom' repository - try { - // role name - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - // look for the custom role - File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName); - System.out.println("** CUSTOM ** " + src); - if(src.exists()) { - // do copy - System.out.println("copying CUSTOM role"); - File dest = new File(ansibleRoot+"/work/"+roleName); - FileUtils.copyDirectory(src, dest); - } - } catch(IOException e) { - e.printStackTrace(); - } - } - } - */ - - public Collection getRoles(CustomRoleManager crm) { - Collection out = new Vector<>(); -// for(String mode:new String[]{"add", "remove", "update"}) { - for(String mode:new String[]{"add"}) { // "remove", "update" - // role name - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); - try { - // look for custom role - Role role = crm.getRole(roleName); - if(role!=null) { - out.add(role); - } - } catch (NoSuchElementException e) { -// e.printStackTrace(); - System.out.println("WARNING: no custom role found for " + roleName); - } - } - return out; - } - - - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java deleted file mode 100644 index e33d6e8..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.io.File; -import java.io.IOException; -import java.util.NoSuchElementException; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - -public class CustomRoleManager { - - private String root; - - public CustomRoleManager(String root) { - this.root = root; - } - - public String getRoot() { - return this.root; - } - - public Role getRole(String roleName) throws NoSuchElementException { - File f = new File(this.getRoot(), roleName); - try { - return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - } catch (IOException e) { -// e.printStackTrace(); - throw new NoSuchElementException("unable to find " + roleName); - } - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java deleted file mode 100644 index 26f0c03..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class DependencyPackage { - - private Dependency dependency; - - public DependencyPackage(Dependency d) { - this.dependency = d; - } - - protected Map getDictionary(Dependency d) { - Map out = new HashMap(); - out.put("name", d.getName()); - out.put("type", d.getType()); - return out; - } - - protected Dependency getDependency() { - return this.dependency; - } - - public Collection getRoles(TemplateManager tm) { - Collection out = new Vector<>(); - for(String mode:new String[]{"add"}) { // "remove", "update" - String roleName = this.getDependency().getType()+"-"+this.getDependency().getName().replaceAll("/", "-")+("add".equals(mode) ? "" : "-"+mode); - try { - // find template - Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode); - // - if(template!=null) { - Map dictionary = this.getDictionary(this.getDependency()); - Role r = tm.fillRoleTemplate(template, dictionary); - r.setName(roleName); - out.add(r); - } else { - System.out.println("WARNING: template is null"); - } - } catch (NoSuchElementException e) { -// e.printStackTrace(); - System.out.println("WARNING: no template found for " + roleName); - } - } - return out; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java deleted file mode 100644 index 02fb3d2..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java +++ /dev/null @@ -1,11 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class OSDependencyPackage extends DependencyPackage { - - public OSDependencyPackage(Dependency d) { - super(d); - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java deleted file mode 100644 index 3c4607f..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - - import java.io.File; -import java.io.IOException; -import java.util.Collection; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; - - public class StaticRoleManager { - - private String root; - - public StaticRoleManager(String root) { - this.root = root; - } - - public String getRoot() { - return this.root; - } - - public Collection getStaticRoles() { - Collection out = new Vector<>(); - for(File f: new File(this.getRoot()).listFiles()) { - try { - out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f)); - } catch(IOException e) { - e.printStackTrace(); - } - } - return out; - } - - - - } diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java deleted file mode 100644 index 73fbf2c..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java +++ /dev/null @@ -1,95 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; - -import java.io.File; -import java.io.IOException; -import java.util.Map; -import java.util.NoSuchElementException; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; -import org.stringtemplate.v4.ST; - -public class TemplateManager { - - private String root; - - public TemplateManager(String root) { - this.root = root; - } - - public String getTemplateRoot() { - return this.root; - } - - /** - * Read the given template - * @param templateName - * @return - * @throws IOException - */ -// private String readTemplate(String templateName) throws IOException { -// File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); -// System.out.println("looking for file " + templateFile.getName()); -// String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8"); -// return out; -// } - - /** - * Return the content of the given template - * @param templateName - * @return - * @throws NoSuchElementException if no such template exists - */ -// public String getTemplate(String templateName) throws NoSuchElementException { -// String template = null; -// try { -// template = this.readTemplate(templateName); -// } catch (IOException e) { -// throw new NoSuchElementException(); -// } -// return template; -// } - - public Role fillRoleTemplate(Role template, Map dictionary) { - Role out = new Role(); - out.setName(template.getName()); - for(RoleFile tf:template.getTaskFiles()) { - out.addTaskFile(this.fillTaskTemplate(tf, dictionary)); - } - for(RoleFile tf:template.getMeta()) { - out.addMeta(this.fillTaskTemplate(tf, dictionary)); - } - return out; - } - - private RoleFile fillTaskTemplate(RoleFile template, Map dictionary) { - RoleFile out = new RoleFile(); - out.setName(template.getName()); - out.setContent(this.fillTemplate(template.getContent(), dictionary)); - return out; - } - - private String fillTemplate(String template, Map dictionary) { - if (template != null) { - ST t = new ST(template); - for (String key : dictionary.keySet()) { - t.add(key, dictionary.get(key)); - } - String output = t.render(); - return output; - } - return template; - } - - public Role getRoleTemplate(String roleName) throws NoSuchElementException { - File f = new File(this.getTemplateRoot(), roleName); - try { - return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f); - } catch (IOException e) { -// e.printStackTrace(); - throw new NoSuchElementException("unable to find " + roleName); - } - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java deleted file mode 100644 index b5b7d07..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.clients; - -import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; -import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; - -import java.util.Collection; -import java.util.List; -import java.util.Vector; - -import org.gcube.common.resources.gcore.ServiceEndpoint; -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -import org.gcube.resources.discovery.client.api.DiscoveryClient; -import org.gcube.resources.discovery.client.queries.api.SimpleQuery; - -public class ISClient { - - /** - * Return the list of hosts (dataminers) in a given VRE - * - * @param vreName - * @return - */ - public Collection listDataminersInVRE() { - - boolean remote = false; - - if (!remote) { - Collection out = new Vector<>(); - Host h = new Host(); - //h.setName("bb-dataminer.res.eng.it"); - //h.setName("vm101.ui.savba.sk"); - h.setName("dataminer1-devnext.d4science.org"); - out.add(h); - return out; - } else { - - SimpleQuery query = queryFor(ServiceEndpoint.class); - - //old version - //query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'") - //.addCondition("$resource/Profile/Name/text() eq 'DataMiner'"); - - query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'"); - - DiscoveryClient client = clientFor(ServiceEndpoint.class); - - List resources = client.submit(query); - - Collection out = new Vector<>(); - for (ServiceEndpoint r : resources) { - Host h = new Host(); - h.setName(r.profile().runtime().hostedOn()); - out.add(h); - } - return out; - } - } - - public static void main(String[] args) { - ISClient a = new ISClient(); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(a.listDataminersInVRE()); -} -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java deleted file mode 100644 index cbccd2c..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java +++ /dev/null @@ -1,35 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Action { - - private String name; - - private String description; - - private String script; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getScript() { - return script; - } - - public void setScript(String script) { - this.script = script; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java deleted file mode 100644 index da57d44..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class Algorithm { - - private String name; - - private String description; - - private String category; - - private String clazz; - - private String algorithmType; - - private String skipJava; - - private String packageURL; - - private Collection actions; - - private Collection dependencies; - - public Algorithm() { - this.actions = new Vector<>(); - this.dependencies = new Vector<>(); - Dependency p = new Dependency(); - } - - public void addDependency(Dependency dep) { - this.dependencies.add(dep); - } - - public void addAction(Action action) { - this.actions.add(action); - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public String getCategory() { - return category; - } - - public void setCategory(String category) { - this.category = category; - } - - public Collection getActions() { - return actions; - } - - public Collection getDependencies() { - return dependencies; - } - - public void setDependencies(Collection deps) { - this.dependencies = deps; - } - - public String toString() { - String out = "Algorithm: " + this.getName()+"\n"; - out+=" Class Name: " + this.getClazz()+"\n"; - out+=" Description: " + this.getDescription()+"\n"; - out+=" Dependencies: " + this.getDependencies()+"\n"; - return out; - } - - public String getClazz() { - return clazz; - } - - public void setClazz(String clazz) { - this.clazz = clazz; - } - - public String getPackageURL() { - return packageURL; - } - - public void setPackageURL(String packageURL) { - this.packageURL = packageURL; - } - - public String getAlgorithmType() { - return algorithmType; - } - - public void setAlgorithmType(String algorithmType) { - this.algorithmType = algorithmType; - } - - public String getSkipJava() { - return skipJava; - } - - public void setSkipJava(String skipJava) { - this.skipJava = skipJava; - } - - - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java deleted file mode 100644 index f85572a..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class AlgorithmSet { - - private String name; - - private Collection algorithms; - - public AlgorithmSet() { - this.algorithms = new Vector<>(); - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Collection getAlgorithms() { - return new Vector<>(algorithms); - } - - public void addAlgorithm(Algorithm algoritm) { - this.algorithms.add(algoritm); - } - - public Boolean hasAlgorithm(Algorithm algorithm) { - for (Algorithm a : this.algorithms) { - if (a.getName().equals(algorithm.getName())) { - return true; - } - } - return false; - } - - public String toString() { - String out = "ALGOSET: " + this.name + "\n"; - for(Algorithm a:this.algorithms) { - out+=a+"\n"; - } - return out; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java deleted file mode 100644 index 8409b40..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java +++ /dev/null @@ -1,73 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -import java.util.Collection; -import java.util.Vector; - -public class Cluster { - - /** - * The set of hosts belonging to the cluster. - */ - private Collection hosts; - - /** - * A name for this cluster. - */ - private String name; - - /** - * A description of this cluster. - */ - private String description; - - /** - * The set of algorithms deployed on this cluster (i.e. on all its hosts) - */ - private Collection algoSets; - - public Cluster() { - this.hosts = new Vector<>(); - this.algoSets = new Vector<>(); - } - - public void addAlgorithmSet(AlgorithmSet set) { - this.algoSets.add(set); - } - - public void addHost(Host host) { - this.hosts.add(host); - } - - public Collection getHosts() { - return hosts; - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getDescription() { - return description; - } - - public void setDescription(String description) { - this.description = description; - } - - public Collection getAlgorithmSets() { - return algoSets; - } - - public String toString() { - String out = "Cluster: "+this.name+"\n"; - for(Host h:this.getHosts()) { - out+=" "+h+"\n"; - } - return out; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java deleted file mode 100644 index 7f9616e..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Dependency { - - private String name; - - private String type; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - - public String toString() { - return this.type+":"+this.name; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java deleted file mode 100644 index 49f463b..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Domain { - - private String name; - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java deleted file mode 100644 index 50eec21..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java +++ /dev/null @@ -1,39 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; - -public class Host { - - private String name; - - private Domain domain; - - public Host() { - } - - public String getFullyQualifiedName() { - if(this.domain!=null && this.domain.getName()!=null) - return this.getName()+"."+this.getDomain().getName(); - else - return this.getName(); - } - - public String getName() { - return name; - } - - public void setName(String name) { - this.name = name; - } - - public Domain getDomain() { - return domain; - } - - public void setDomain(Domain domain) { - this.domain = domain; - } - - public String toString() { - return this.name + "@" + this.domain; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java deleted file mode 100644 index 024c2d5..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java +++ /dev/null @@ -1,15 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; - -public class AlgorithmComparator implements Comparator { - - @Override - public int compare(Algorithm a1, Algorithm a2) { - return a1.getName().compareTo(a2.getName()); - } - - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java deleted file mode 100644 index abbfbbd..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java +++ /dev/null @@ -1,18 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class DependencyComparator implements Comparator { - - @Override - public int compare(Dependency a1, Dependency a2) { - int out = a1.getType().compareTo(a2.getType()); - if(out!=0) - return out; - return a1.getName().compareTo(a2.getName()); - } - - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java deleted file mode 100644 index 1e159b8..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator; - -import java.util.Comparator; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; - -public class HostComparator implements Comparator { - - @Override - public int compare(Host h1, Host h2) { - int out = h1.getDomain().getName().compareTo(h2.getDomain().getName()); - if(out!=0) - return out; - return h1.getName().compareTo(h2.getName()); - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java deleted file mode 100644 index b6b3c27..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java +++ /dev/null @@ -1,103 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.process; - -import java.util.StringTokenizer; - -public class AddAlgorithmCommand { - - private String command; - private String name; - private String category; - private String clazz; - private String scope; - private String algorithmType; - private String skipJava; - private String url; - private String description; - - public AddAlgorithmCommand(String cmd) { - StringTokenizer st = new StringTokenizer(cmd, " "); - if (st.hasMoreElements()) - command = st.nextToken(); - if (st.hasMoreElements()) - name = st.nextToken(); - if (st.hasMoreElements()) - category = st.nextToken(); - if (st.hasMoreElements()) - clazz = st.nextToken(); - if (st.hasMoreElements()) - scope = st.nextToken(); - if (st.hasMoreElements()) - algorithmType = st.nextToken(); - if (st.hasMoreElements()) - skipJava = st.nextToken(); - if (st.hasMoreElements()) - url = st.nextToken(); - - String d = ""; - while (st.hasMoreElements()) - d = d + st.nextToken() + " "; - this.setDescription(d); - - } - - public void setDescription(String d) { - if(d!=null) { - d = d.trim(); - if(d.startsWith("\"") && d.endsWith("\"")) { - d = d.substring(1, d.length()-1).trim(); - } - } - this.description = d; - } - - public String getCommand() { - return command; - } - - public String getName() { - return name; - } - - public String getCategory() { - return category; - } - - public String getClazz() { - return clazz; - } - - public String getVRE() { - return scope; - } - - public String getAlgorithmType() { - return algorithmType; - } - - public String getSkipjava() { - return skipJava; - } - - public String getUrl() { - return url; - } - - public String getDescription() { - return description; - } - - public String toString() { - String out = ""; - out += String.format("%-12s: %s\n", "command", command); - out += String.format("%-12s: %s\n", "algo name", name); - out += String.format("%-12s: %s\n", "category", category); - out += String.format("%-12s: %s\n", "class", clazz); - out += String.format("%-12s: %s\n", "scope", scope); - out += String.format("%-12s: %s\n", "algo type", algorithmType); - out += String.format("%-12s: %s\n", "skip java", skipJava); - out += String.format("%-12s: %s\n", "url", url); - out += String.format("%-12s: %s\n", "description", this.description); - return out; - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java deleted file mode 100644 index eb87630..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ /dev/null @@ -1,290 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.process; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Vector; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import org.apache.commons.lang.StringEscapeUtils; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; - -public class AlgorithmPackageParser { - - /** - * The name of the file containing algorithm metadata. Expected in the root - * directory of the package. - */ - private static final String METADATA_FILE_NAME = "Info.txt"; - - private static final String METADATA_ALGORITHM_NAME = "Algorithm Name"; - - private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description"; - - private static final String METADATA_CLASS_NAME = "Class Name"; - - private static final String METADATA_PACKAGES = "Packages"; - - private static final String METADATA_KEY_VALUE_SEPARATOR = ":"; - - private static final int BUFFER_SIZE = 4096; - - /** - * Given an URL to an algorithm package, create an Algorithm object with its - * metadata. Metadata are extracted from the 'info.txt' file, if any, in the - * package. - * - * @param url - * @return An Algorithm object or null if no 'info.txt' is found in the - * package. - * @throws IOException - */ - public Algorithm parsePackage(String url) throws IOException { - String packageMetadata = this.getPackageMetadata(url); - if (packageMetadata == null) { - System.out.println("WARNING: No metadata found for " + url); - return null; - } else { - Map> parsedMetadata = this.parseMetadata(packageMetadata); - Algorithm a = this.createAlgorithm(parsedMetadata); - a.setPackageURL(url); - return a; - } - } - - /** - * Extract the content of the metadata file from the package. - * - * @param url - * @return - * @throws IOException - */ - private String getPackageMetadata(String url) throws IOException { - InputStream is = new URL(url).openStream(); - ZipInputStream zipIs = new ZipInputStream(is); - ZipEntry entry = zipIs.getNextEntry(); - String out = null; - while (entry != null) { - if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) { - out = this.getEntryContent(zipIs); - break; - } - entry = zipIs.getNextEntry(); - } - is.close(); - zipIs.close(); - return out; - } - - /** - * Read the content of a zip entry and place it in a string. - * @param zipIn - * @return - * @throws IOException - */ - private String getEntryContent(ZipInputStream zipIn) throws IOException { - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = zipIn.read(buffer)) != -1) { - s.append(new String(buffer, 0, read)); - } - return s.toString(); - } - - /** - * Parse the content of the metadata file and create a key+multivalue map. - * @param metadata - * @return - */ - private Map> parseMetadata(String metadata) { - Map> out = new HashMap>(); - String[] lines = metadata.split("\n"); - - String key = null; - String value = null; - - for (String line : lines) { - // skip empty lines - if (line.trim().isEmpty()) { - continue; - } - // scan lines one by one, looking for key and values - String[] parts = line.split(METADATA_KEY_VALUE_SEPARATOR); - if (parts.length > 1) { - // key and value on the same line - key = parts[0].trim(); - value = line.substring(parts[0].length() + 1).trim(); - } else if (parts.length == 1) { - // either a key or a value - if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) { - // key - key = parts[0].trim(); - value = null; - } else { - // value - value = line.trim(); - } - } - // add key+value to the map - if (key != null && value != null) { - List values = out.get(key); - if (values == null) { - values = new Vector<>(); - out.put(key, values); - } - values.add(value); - System.out.println(key + METADATA_KEY_VALUE_SEPARATOR + " " + values); - } - } - return out; - } - - /** - * Create an Algorithm starting from its metadata - * @param metadata - * @return - */ -// private Algorithm createAlgorithm(Map> metadata) { -// Algorithm out = new Algorithm(); -// out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); -// out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); -// out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); -// List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); -// if (dependencies != null) { -// for (String pkg : dependencies) { -// Dependency dep = new Dependency(); -// dep.setName(pkg); -// dep.setType("os"); -// out.addDependency(dep); -// } -// } -// return out; -// } - - - private Algorithm createAlgorithm(Map> metadata) { - Algorithm out = new Algorithm(); - out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); - out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION)); - out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); - //List dependencies = extractMultipleValues(metadata, METADATA_PACKAGES); - - - List rdependencies = extractMultipleValues(metadata, "cran"); - if (rdependencies != null) { - for (String pkg : rdependencies) { - Dependency dep = new Dependency(); - - //if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("cran"); - out.addDependency(dep); - } - } - - - List defdependencies = extractMultipleValues(metadata, "Packages"); - if (defdependencies != null) { - for (String pkg : defdependencies) { - Dependency dep = new Dependency(); - - //if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - } - } - - List osdependencies = extractMultipleValues(metadata, "os"); - if (osdependencies != null) { - for (String pkg : osdependencies) { - Dependency dep = new Dependency(); - - //if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("os"); - out.addDependency(dep); - } - } - - - - List gitdependencies = extractMultipleValues(metadata, "github"); - if (gitdependencies != null) { - for (String pkg : gitdependencies) { - Dependency dep = new Dependency(); - - //if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("github"); - out.addDependency(dep); - } - } - - - - List cdependencies = extractMultipleValues(metadata, "custom"); - if (cdependencies != null) { - for (String pkg : cdependencies) { - Dependency dep = new Dependency(); - - //if (pkg.startsWith("os:")){ - dep.setName(pkg); - dep.setType("custom"); - out.addDependency(dep); - } - } - - -// if (pkg.startsWith("r:")){ -// //String results = StringEscapeUtils.escapeJava(pkg); -// dep.setName(pkg); -// dep.setType("cran"); -// } -// if (pkg.startsWith("custom:")){ -// dep.setName(pkg); -// dep.setType("custom"); -// } -// if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:")){ -// dep.setName(pkg); -// dep.setType("os"); -// } - - - - return out; - } - - - - - private static String extractSingleValue(Map> metadata, - String key) { - List l = metadata.get(key); - if (l != null && l.size() == 1) { - return l.get(0); - } else { - return null; - } - } - - - - - private static List extractMultipleValues( - Map> metadata, String key) { - List l = metadata.get(key); - if (l != null) { - return new Vector<>(l); - } else { - return null; - } - } - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java deleted file mode 100644 index 600c188..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.rest; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; - -public interface PoolManager { - - String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException; - - Algorithm extractAlgorithm(String url) throws IOException; - - String getLogById(String logId) throws IOException; - - void getLogId(Algorithm algo, String vre); - - String getScriptFromURL(URL logId) throws IOException; - - URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException; - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java deleted file mode 100644 index 94c2081..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ /dev/null @@ -1,144 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.rest; - -import java.io.IOException; -import java.net.MalformedURLException; -import java.net.ProxySelector; -import java.net.URL; -import java.net.UnknownHostException; - -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - - - - -@Path("/") -public class RestPoolManager implements PoolManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class); - - private PoolManager service = new DataminerPoolManager(); - - - @GET - @Path("/add") - @Produces("text/plain") - public String addAlgorithmToVRE( - @QueryParam("algorithm") String algorithm, - @QueryParam("vre") String vre, - @QueryParam("name") String name, - @QueryParam("description") String description, - @QueryParam("category") String category, - @QueryParam("algorithmType") String algorithmType, - @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException { - // TODO Auto-generated method stub - LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre); - Algorithm algo = service.extractAlgorithm(algorithm); - - if (algo.getCategory() == null){ - algo.setCategory(category); - } else category = algo.getCategory(); - - if (algo.getAlgorithmType() == null){ - algo.setAlgorithmType(algorithmType); - } else algorithmType = algo.getCategory(); - - if (algo.getSkipJava() == null){ - algo.setSkipJava(skipJava); - } else skipJava = algo.getSkipJava(); - - if (algo.getName() == null){ - algo.setCategory(name); - } else name = algo.getName(); - - if (algo.getDescription() == null){ - algo.setDescription(description);; - } else description = algo.getDescription(); - - return service.addAlgorithmToVRE(algo, vre); - } - - - @GET - @Path("/log") - @Produces("text/plain") - public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException { - // TODO Auto-generated method stub - LOGGER.debug("Returning Log =" + logUrl); - return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl)); - } - - - - - - @Override - public Algorithm extractAlgorithm(String url) throws IOException { - // TODO Auto-generated method stub - return null; - } - - - public static void main(String[] args) throws IOException, InterruptedException { - RestPoolManager a = new RestPoolManager(); - //System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); - - - //ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - a.addAlgorithmToVRE( - "http://data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0", - "/gcube/devNext/NextNext", - null, null, "test", "transducerers", "N"); - - //System.out.println(a.getLogById("34ac474d-b9df-4929-87e1-2a0ae26cf898")); - } - - - @Override - public void getLogId(Algorithm algo, String vre) { - // TODO Auto-generated method stub - - } - - - @Override - public String getScriptFromURL(URL logId) throws IOException { - // TODO Auto-generated method stub - return null; - } - - - @Override - public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException { - // TODO Auto-generated method stub - return null; - } - - - @Override - public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { - // TODO Auto-generated method stub - return null; - } - - - - - - - - - - -} \ No newline at end of file diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java deleted file mode 100644 index 863237b..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ /dev/null @@ -1,262 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.service; - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.URLConnection; -import java.net.UnknownHostException; -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class DataminerPoolManager implements PoolManager { - - - private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); - - -// static Collection algorithms; -// -// static Collection sets; -// -// static { -// algorithms = new Vector<>(); -// } -// -// public DataminerPoolManager() { -// } -// -// /** -// * Add a new algorithm to the set of known ones. No further action is expected -// * on the pool. -// */ -// public void publishAlgorithm(Algorithm algorithm) { -// algorithms.add(algorithm); -// } -// -// /** -// * Re-deploy the given algorithm wherever it's installed -// * -// * @param algorithm -// */ -// /* -// * public void updateAlgorithm(Algorithm algorithm) { // TODO implement this } -// */ -// -// /** -// * Add the give algorithm to the given set -// * -// * @param algorithmId -// * @param setId -// */ -// public void addAlgorithmToSet(String algorithmName, String setName) { -// AlgorithmSet set = this.getAlgorithmSet(setName); -// Algorithm algorithm = this.getAlgorithm(algorithmName); -// if (set != null && algorithm != null) { -// set.addAlgorithm(algorithm); -// this.updateClusters(); -// } -// } -// -// /** -// * Apply the given set of algorithms to the given cluster -// * -// * @param setId -// * @param clusterId -// */ -// public void applyAlgorithmSetToCluster(String setName, String clusterName) { -// AlgorithmSet set = this.getAlgorithmSet(setName); -// Cluster cluster = new ISClient().getCluster(clusterName); -// if (set != null && cluster != null) { -// cluster.addAlgorithmSet(set); -// this.updateClusters(); -// } -// } -// -// private AlgorithmSet getAlgorithmSet(String name) { -// for (AlgorithmSet set : sets) { -// if (name.equals(set.getName())) { -// return set; -// } -// } -// return null; -// } -// -// private Algorithm getAlgorithm(String name) { -// for (Algorithm a : algorithms) { -// if (name.equals(a.getName())) { -// return a; -// } -// } -// return null; -// } - - - - -// -// public void getLogId(final Algorithm algorithm, final String vre) { -// new Thread() { -// public void run() { -// while (true) { -// try { -// addAlgorithmToVRE(algorithm, vre); -// } catch (Exception e) { -// //log here -// } -// } -// } -// }.start(); -// } -// -// - -// public String getLogId(){ -// PrintStream console = System.out; -// File path = new File(worker.getWorkdir() + File.separator + "logs"); -// path.mkdirs(); -// File n = new File(path + File.separator + worker.getWorkerId()); -// FileOutputStream fos = new FileOutputStream(n); -// PrintStream ps = new PrintStream(fos); -// System.setOut(ps); -// worker.apply(); -// System.setOut(console); -// worker.apply(); -// System.out.println("Log stored to to " + n.getAbsolutePath()); -// } - - - - -// public String getLogById(String id) throws IOException { -// String strLine = null; -// try{ -// FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id); -// BufferedReader br = new BufferedReader(new InputStreamReader(fstream)); -// /* read log line by line */ -// while ((strLine = br.readLine()) != null) { -// /* parse strLine to obtain what you want */ -// System.out.println (strLine); -// } -// br.close(); -// } catch (Exception e) { -// System.err.println("Error: " + e.getMessage()); -// } -// return strLine; -// } - - - public String getScriptFromURL(URL url) throws IOException { - if (url == null) { - return null; - } - URLConnection yc = url.openConnection(); - BufferedReader input = new BufferedReader(new InputStreamReader( - yc.getInputStream())); - String line; - StringBuffer buffer = new StringBuffer(); - while ((line = input.readLine()) != null) { - buffer.append(line + "\n"); - } - String bufferScript = buffer.substring(0, buffer.length()); - input.close(); - return bufferScript; - } - - - - - - /** - * Publish the given algorithm in the given VRE - * - * @param algorithmName - * @param vre - * - */ - public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException { - // create a fake algorithm set - final AlgorithmSet algoSet = new AlgorithmSet(); - algoSet.setName("fake"); - algoSet.addAlgorithm(algorithm); - final String uuid = UUID.randomUUID().toString(); - - new Thread(new Runnable() { - @Override - public void run() { - // TODO Auto-generated method stub - try { - addAlgorithmsToVRE(algoSet, vre, uuid); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - }).start(); - //this line will execute immediately, not waiting for your task to complete - System.out.println(uuid); - return uuid; - } - - - - public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{ - - File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs"); - path.mkdirs(); - File n = new File(path + File.separator +a); - //String addr = InetAddress.getLocalHost().getHostAddress(); - - return new File(n.getPath()).toURI().toURL(); - } - - - public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException { - - // create the cluster (dataminers in the vre) - Cluster cluster = new Cluster(); - for(Host h:new ISClient().listDataminersInVRE()) { - cluster.addHost(h); - } - - // apply the changes - AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId(); - - } - - public Algorithm extractAlgorithm(String url) throws IOException { - return new AlgorithmPackageParser().parsePackage(url); - } - - -@Override -public void getLogId(Algorithm algo, String vre) { - // TODO Auto-generated method stub - -} - - -@Override -public String getLogById(String logId) throws IOException { - // TODO Auto-generated method stub - return null; -} - - - - - -} diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java deleted file mode 100644 index 924a995..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java +++ /dev/null @@ -1,145 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.util; - -import java.io.IOException; -import java.net.Authenticator; -import java.net.InetSocketAddress; -import java.net.PasswordAuthentication; -import java.net.Proxy; -import java.net.ProxySelector; -import java.net.SocketAddress; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.configuration.ConfigurationException; -import org.apache.commons.configuration.PropertiesConfiguration; -import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy; - -interface NetworkConfiguration { - - public String getProxyHost(); - - public String getProxyPort(); - - public String getProxyUser(); - - public String getProxyPassword(); - - public String getNonProxyHosts(); - -} - -class FileBasedProxyConfiguration implements NetworkConfiguration { - - private static PropertiesConfiguration configuration; - - public FileBasedProxyConfiguration(String path) { - try { - // load the configuration - configuration = new PropertiesConfiguration(path); - // set the reloading strategy to enable hot-configuration - FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy(); - configuration.setReloadingStrategy(fcrs); - } catch (ConfigurationException e) { - e.printStackTrace(); - } - } - - @Override - public String getProxyHost() { - return configuration.getString("proxyHost"); - } - - @Override - public String getProxyPort() { - return configuration.getString("proxyPort"); - } - - @Override - public String getProxyUser() { - return configuration.getString("proxyUser"); - } - - @Override - public String getProxyPassword() { - return configuration.getString("proxyPassword"); - } - - @Override - public String getNonProxyHosts() { - return configuration.getString("nonProxyHosts"); - } - -} - -public class PropertiesBasedProxySelector extends ProxySelector { - - List proxies = null; - - List nonProxyHosts = null; - - public PropertiesBasedProxySelector(String proxySettingsPath) { - this(new FileBasedProxyConfiguration(proxySettingsPath)); - } - - public PropertiesBasedProxySelector(NetworkConfiguration config) { - if (config == null || config.getProxyHost() == null) { - this.proxies = null; - return; - } - - String host = config.getProxyHost(); - - int port = 80; - - if (config.getProxyPort() != null) { - port = Integer.valueOf(config.getProxyPort()); - } - - if (config.getNonProxyHosts() != null) { - this.nonProxyHosts = Arrays - .asList(config.getNonProxyHosts().split("\\|")); - } - - this.proxies = new ArrayList(); - this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host, - port))); - - if (config.getProxyUser() != null) { - final String username = config.getProxyUser(); - final String password = config.getProxyPassword(); - - Authenticator.setDefault(new Authenticator() { - @Override - protected PasswordAuthentication getPasswordAuthentication() { - return new PasswordAuthentication(username, password.toCharArray()); - } - }); - - } - } - - @Override - public List select(URI uri) { - if (this.nonProxyHosts == null) { - return Arrays.asList(Proxy.NO_PROXY); - } else { - for (String entry : this.nonProxyHosts) { - entry = entry.trim(); - if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) { - return Arrays.asList(Proxy.NO_PROXY); - } - if (uri.getHost().equals(entry)) { - return Arrays.asList(Proxy.NO_PROXY); - } - } - return this.proxies; - } - } - - @Override - public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) { - - } -} diff --git a/src/main/webapp/WEB-INF/gcube-app.xml b/src/main/webapp/WEB-INF/gcube-app.xml deleted file mode 100644 index 15f231c..0000000 --- a/src/main/webapp/WEB-INF/gcube-app.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - dataminer-pool-manager - dataanalysis - 0.0.1 - - - - - - - - \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/gcube-handlers.xml b/src/main/webapp/WEB-INF/gcube-handlers.xml deleted file mode 100644 index c294252..0000000 --- a/src/main/webapp/WEB-INF/gcube-handlers.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml deleted file mode 100644 index 4c42390..0000000 --- a/src/main/webapp/WEB-INF/web.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - Archetype Created Web Application - - - - REST-API - org.glassfish.jersey.servlet.ServletContainer - - jersey.config.server.provider.packages - org.gcube.dataanalysis.dataminer.poolmanager.rest - - 1 - - - - REST-API - /rest/* - - - diff --git a/src/test/java/AlgorithmPackageParserTest.java b/src/test/java/AlgorithmPackageParserTest.java deleted file mode 100644 index b3ea7bb..0000000 --- a/src/test/java/AlgorithmPackageParserTest.java +++ /dev/null @@ -1,117 +0,0 @@ - - -import java.io.IOException; -import java.io.InputStream; -import java.net.URL; -import java.util.List; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; - -public class AlgorithmPackageParserTest { - - private static int BUFFER_SIZE = 2048; - - public void extractAllAlgorithms() throws IOException { - String url = "http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; - List commands = this.extractAddAlgorithmCommands(url); - AlgorithmSet algorithms = new AlgorithmSet(); - for (String cmd : commands) { - System.out.println("-----------------------------------------"); - System.out.println(cmd); - AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - System.out.println(aac); - - // start creating the algo from the command - Algorithm algo = new Algorithm(); - algo.setAlgorithmType(aac.getAlgorithmType()); - algo.setCategory(aac.getCategory()); - algo.setClazz(aac.getClazz()); - algo.setDescription(aac.getDescription()); - algo.setName(aac.getName()); - algo.setPackageURL(aac.getUrl()); - algo.setSkipJava(aac.getSkipjava()); - - // then override with info from the package - if (aac.getUrl().length() > 4) { - Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); - if (packagedAlgo != null) { - algo.setDependencies(packagedAlgo.getDependencies()); - } - } - algorithms.addAlgorithm(algo); - break; - } - //to uncomment - // new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - // "/gcube/devNext/NextNext"); - } - - /** - * Extract 'addAlgorithm' commands from a file containing wiki-table-style - * entries for algorithm. - * - * @return - * @throws IOException - */ - private List extractAddAlgorithmCommands(String listUrl) - throws IOException { - URL url = new URL(listUrl); - InputStream is = url.openStream(); - - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = is.read(buffer)) != -1) { - s.append(new String(buffer, 0, read)); - } - List out = new Vector<>(); - String[] lines = s.toString().split("\n"); - for (String line : lines) { - System.out.println("--------------------"); - if (!line.isEmpty()) { - String[] parts = line.split("\\|"); - int c = 1; - for (String part : parts) { - if (part == null || part.trim().isEmpty()) { - continue; - } - System.out.println(c + ". " + part); - c++; - if (part.contains("addAlgorithm.sh")) { - String cmd = part.trim(); - cmd = cmd.replaceAll("", ""); - cmd = cmd.replaceAll("", ""); - System.out.println(cmd); - // AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - // System.out.println(aac); - out.add(cmd); - } - } - } - } - return out; - } - - /** - * Create an Algorithm starting from the algorithm jar. - * - * @param url - * @return - * @throws IOException - */ - private Algorithm extractAlgorithm(String url) throws IOException { - return new AlgorithmPackageParser().parsePackage(url); - } - - public static void main(String[] args) throws Exception { -// ProxySelector.setDefault(new PropertiesBasedProxySelector( -// "/home/ngalante/.proxy-settings")); - - new AlgorithmPackageParserTest().extractAllAlgorithms(); - } - -} diff --git a/src/test/java/AnsibleWorkerTest.java b/src/test/java/AnsibleWorkerTest.java deleted file mode 100644 index 85e5b83..0000000 --- a/src/test/java/AnsibleWorkerTest.java +++ /dev/null @@ -1,33 +0,0 @@ - - -import java.io.File; -import java.io.IOException; -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleWorkerTest { - - public static void main(String[] args) throws IOException { - AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString())); - - System.out.println("created worker named " + worker.getWorkerId()); - - worker.setInventory(new Inventory()); - worker.setPlaybook(new Playbook()); - - Role r = new Role(); - r.setName("latex"); - - RoleFile tf = new RoleFile("main", "do something special for " + r.getName()); - r.addTaskFile(tf); - worker.addRole(r); - - worker.apply(); - } - -} diff --git a/src/test/java/DataminerPoolManagerTest.java b/src/test/java/DataminerPoolManagerTest.java deleted file mode 100644 index 33e2aea..0000000 --- a/src/test/java/DataminerPoolManagerTest.java +++ /dev/null @@ -1,227 +0,0 @@ - - -import java.net.ProxySelector; -import java.util.UUID; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class DataminerPoolManagerTest { - - /* - private static Map domains = new HashMap<>(); - private static Map clusters = new HashMap<>(); - - private static Dependency createDependency(String depName) { - String[] parts = depName.split(":"); - Dependency out = new Dependency(); - if(parts.length>1) { - out.setType(parts[0]); - out.setName(parts[1]); - } else { - out.setType("os"); - out.setName(depName); - } - return out; - } - - private static Algorithm createAlgorithm(String name, String ... deps) { - Algorithm a = new Algorithm(); - a.setName(name); - for(String dep:deps) { - a.addDependency(createDependency(dep)); - } - return a; - } - - private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) { - AlgorithmSet out = new AlgorithmSet(); - out.setName(name); - for(Algorithm a:algs) { - out.addAlgorithm(a); - } - return out; - } - - private static Domain getDomain(String name) { - if(domains.get(name)==null) { - Domain d = new Domain(); - d.setName(name); - domains.put(name, d); - return d; - } else { - return domains.get(name); - } - } - - private static Host createHost(String hostname, String domainName) { - Host out = new Host(); - out.setName(hostname); - Domain d = getDomain(domainName); - out.setDomain(d); - return out; - } - - private static Cluster getCluster(String name) { - if(clusters.get(name)==null) { - Cluster d = new Cluster(); - d.setName(name); - clusters.put(name, d); - return d; - } else { - return clusters.get(name); - } - } - - private static Collection extractDependencies() { - Collection out = new TreeSet<>(new DependencyComparator()); - for(Cluster c:clusters.values()) { - for(AlgorithmSet as:c.getAlgorithmSets()) { - for(Algorithm a:as.getAlgorithms()) { - for(Dependency d:a.getDependencies()) { - out.add(d); - } - } - } - } - return out; - } - - private static Collection extractAlgorithms() { - Collection out = new TreeSet<>(new AlgorithmComparator()); - for(Cluster c:clusters.values()) { - for(AlgorithmSet as:c.getAlgorithmSets()) { - for(Algorithm a:as.getAlgorithms()) { - out.add(a); - } - } - } - return out; - } - - static { - - Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package"); - Algorithm ensemble = createAlgorithm("ensemble", "python"); - Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah"); - - AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe); - AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble); - AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe); - - Cluster cluster1 = getCluster("cluster-1"); - cluster1.addHost(createHost("host1", "domain1")); - cluster1.addHost(createHost("host2", "domain1")); - cluster1.addHost(createHost("host3", "domain1")); - cluster1.addHost(createHost("host1", "domain2")); - cluster1.addHost(createHost("host2", "domain2")); - - Cluster cluster2 = getCluster("cluster-2"); - cluster2.addHost(createHost("host4", "domain1")); - cluster2.addHost(createHost("host5", "domain1")); - cluster2.addHost(createHost("host6", "domain1")); - cluster2.addHost(createHost("host3", "domain2")); - cluster2.addHost(createHost("host4", "domain2")); - cluster2.addHost(createHost("host5", "domain2")); - - cluster1.addAlgorithmSet(as1); - cluster1.addAlgorithmSet(as2); - - cluster2.addAlgorithmSet(as1); - cluster2.addAlgorithmSet(as3); - - } - */ - - public static void main(String[] args) throws Exception { - /* - AnsibleBridge ab = new AnsibleBridge(); - ab.printInventoryByDomainAndSets(clusters.values()); - System.out.println("-----------"); - ab.printInventoryBySets(clusters.values()); - - AnsibleWorker worker = ab.createWorker(); - - for(Algorithm a:extractAlgorithms()) { - for(Role r:ab.generateRoles(a)) { - worker.addRole(r); - } - } - - for(Dependency d:extractDependencies()) { - for(Role r:ab.generateRoles(d)) { - worker.addRole(r); - } - } - */ - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -// -// // create the algorithm (download it, etc etc) -// Algorithm algorithm = new Algorithm(); -// algorithm.setName("ichtyop"); -// algorithm.setClazz("org.gcube..."); -// algorithm.setDescription("some description"); -// -// Dependency d = new Dependency(); -// d.setName("libpng"); -// d.setType("os"); -// algorithm.addDependency(d); -// -// d = new Dependency(); -// d.setName("some-r-package"); -// d.setType("cran"); -// algorithm.addDependency(d); -// -// d = new Dependency(); -// d.setName("some-other-r-package"); -// d.setType("cran"); -// algorithm.addDependency(d); -// -// d = new Dependency(); -// d.setName("voodoo"); -// d.setType("custom"); -// algorithm.addDependency(d); -// -// // create the algorithm (download it, etc etc) -// Algorithm ewe = new Algorithm(); -// ewe.setName("ewe"); -// -// d = new Dependency(); -// d.setName("voodoo"); -// d.setType("custom"); -// ewe.addDependency(d); - - AlgorithmSet algorithms = new AlgorithmSet(); - algorithms.setName("dummy-set"); - - -// algorithms.addAlgorithm(algorithm); -// algorithms.addAlgorithm(ewe); - - Algorithm ensemble = new Algorithm(); - ensemble.setName("ewe"); - ensemble.setCategory("a"); - ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http://data.d4science.org/eDd5b2ovMmpSVEZaZWYvY3g2bDZNcGRFNUIxNi85UFlHbWJQNStIS0N6Yz0"); - ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); - ensemble.setSkipJava("N"); - ensemble.setDescription("test"); - - Dependency d = new Dependency(); - d.setName("libpng3"); - d.setType("os"); - ensemble.addDependency(d); - algorithms.addAlgorithm(ensemble); - - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID()); - - } - -} diff --git a/src/test/java/ISClientTest.java b/src/test/java/ISClientTest.java deleted file mode 100644 index bdc8cfc..0000000 --- a/src/test/java/ISClientTest.java +++ /dev/null @@ -1,17 +0,0 @@ - - -import java.net.ProxySelector; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class ISClientTest { - - public static void main(String[] args) { - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(new ISClient().listDataminersInVRE()); - } - -} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java deleted file mode 100644 index 1094598..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java +++ /dev/null @@ -1,119 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.io.IOException; -import java.io.InputStream; -import java.net.ProxySelector; -import java.net.URL; -import java.util.List; -import java.util.Vector; - -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand; -import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class AlgorithmPackageParserTest { - - private static int BUFFER_SIZE = 2048; - - public void extractAllAlgorithms() throws IOException { - String url = "http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms"; - List commands = this.extractAddAlgorithmCommands(url); - AlgorithmSet algorithms = new AlgorithmSet(); - for (String cmd : commands) { - System.out.println("-----------------------------------------"); - System.out.println(cmd); - AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - System.out.println(aac); - - // start creating the algo from the command - Algorithm algo = new Algorithm(); - algo.setAlgorithmType(aac.getAlgorithmType()); - algo.setCategory(aac.getCategory()); - algo.setClazz(aac.getClazz()); - algo.setDescription(aac.getDescription()); - algo.setName(aac.getName()); - algo.setPackageURL(aac.getUrl()); - algo.setSkipJava(aac.getSkipjava()); - - // then override with info from the package - if (aac.getUrl().length() > 4) { - Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl()); - if (packagedAlgo != null) { - algo.setDependencies(packagedAlgo.getDependencies()); - } - } - algorithms.addAlgorithm(algo); - break; - } - //to uncomment - // new DataminerPoolManager().addAlgorithmsToVRE(algorithms, - // "/gcube/devNext/NextNext"); - } - - /** - * Extract 'addAlgorithm' commands from a file containing wiki-table-style - * entries for algorithm. - * - * @return - * @throws IOException - */ - private List extractAddAlgorithmCommands(String listUrl) - throws IOException { - URL url = new URL(listUrl); - InputStream is = url.openStream(); - - StringBuilder s = new StringBuilder(); - byte[] buffer = new byte[BUFFER_SIZE]; - int read = 0; - while ((read = is.read(buffer)) != -1) { - s.append(new String(buffer, 0, read)); - } - List out = new Vector<>(); - String[] lines = s.toString().split("\n"); - for (String line : lines) { - System.out.println("--------------------"); - if (!line.isEmpty()) { - String[] parts = line.split("\\|"); - int c = 1; - for (String part : parts) { - if (part == null || part.trim().isEmpty()) { - continue; - } - System.out.println(c + ". " + part); - c++; - if (part.contains("addAlgorithm.sh")) { - String cmd = part.trim(); - cmd = cmd.replaceAll("", ""); - cmd = cmd.replaceAll("", ""); - System.out.println(cmd); - // AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd); - // System.out.println(aac); - out.add(cmd); - } - } - } - } - return out; - } - - /** - * Create an Algorithm starting from the algorithm jar. - * - * @param url - * @return - * @throws IOException - */ - private Algorithm extractAlgorithm(String url) throws IOException { - return new AlgorithmPackageParser().parsePackage(url); - } - - public static void main(String[] args) throws Exception { -// ProxySelector.setDefault(new PropertiesBasedProxySelector( -// "/home/ngalante/.proxy-settings")); - - new AlgorithmPackageParserTest().extractAllAlgorithms(); - } - -} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java deleted file mode 100644 index 77f02b5..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.io.File; -import java.io.IOException; -import java.util.UUID; - -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; -import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; - -public class AnsibleWorkerTest { - - public static void main(String[] args) throws IOException { - AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString())); - - System.out.println("created worker named " + worker.getWorkerId()); - - worker.setInventory(new Inventory()); - worker.setPlaybook(new Playbook()); - - Role r = new Role(); - r.setName("latex"); - - RoleFile tf = new RoleFile("main", "do something special for " + r.getName()); - r.addTaskFile(tf); - worker.addRole(r); - - worker.apply(); - } - -} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java deleted file mode 100644 index d2af50f..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java +++ /dev/null @@ -1,227 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.net.ProxySelector; -import java.util.UUID; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; -import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class DataminerPoolManagerTest { - - /* - private static Map domains = new HashMap<>(); - private static Map clusters = new HashMap<>(); - - private static Dependency createDependency(String depName) { - String[] parts = depName.split(":"); - Dependency out = new Dependency(); - if(parts.length>1) { - out.setType(parts[0]); - out.setName(parts[1]); - } else { - out.setType("os"); - out.setName(depName); - } - return out; - } - - private static Algorithm createAlgorithm(String name, String ... deps) { - Algorithm a = new Algorithm(); - a.setName(name); - for(String dep:deps) { - a.addDependency(createDependency(dep)); - } - return a; - } - - private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) { - AlgorithmSet out = new AlgorithmSet(); - out.setName(name); - for(Algorithm a:algs) { - out.addAlgorithm(a); - } - return out; - } - - private static Domain getDomain(String name) { - if(domains.get(name)==null) { - Domain d = new Domain(); - d.setName(name); - domains.put(name, d); - return d; - } else { - return domains.get(name); - } - } - - private static Host createHost(String hostname, String domainName) { - Host out = new Host(); - out.setName(hostname); - Domain d = getDomain(domainName); - out.setDomain(d); - return out; - } - - private static Cluster getCluster(String name) { - if(clusters.get(name)==null) { - Cluster d = new Cluster(); - d.setName(name); - clusters.put(name, d); - return d; - } else { - return clusters.get(name); - } - } - - private static Collection extractDependencies() { - Collection out = new TreeSet<>(new DependencyComparator()); - for(Cluster c:clusters.values()) { - for(AlgorithmSet as:c.getAlgorithmSets()) { - for(Algorithm a:as.getAlgorithms()) { - for(Dependency d:a.getDependencies()) { - out.add(d); - } - } - } - } - return out; - } - - private static Collection extractAlgorithms() { - Collection out = new TreeSet<>(new AlgorithmComparator()); - for(Cluster c:clusters.values()) { - for(AlgorithmSet as:c.getAlgorithmSets()) { - for(Algorithm a:as.getAlgorithms()) { - out.add(a); - } - } - } - return out; - } - - static { - - Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package"); - Algorithm ensemble = createAlgorithm("ensemble", "python"); - Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah"); - - AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe); - AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble); - AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe); - - Cluster cluster1 = getCluster("cluster-1"); - cluster1.addHost(createHost("host1", "domain1")); - cluster1.addHost(createHost("host2", "domain1")); - cluster1.addHost(createHost("host3", "domain1")); - cluster1.addHost(createHost("host1", "domain2")); - cluster1.addHost(createHost("host2", "domain2")); - - Cluster cluster2 = getCluster("cluster-2"); - cluster2.addHost(createHost("host4", "domain1")); - cluster2.addHost(createHost("host5", "domain1")); - cluster2.addHost(createHost("host6", "domain1")); - cluster2.addHost(createHost("host3", "domain2")); - cluster2.addHost(createHost("host4", "domain2")); - cluster2.addHost(createHost("host5", "domain2")); - - cluster1.addAlgorithmSet(as1); - cluster1.addAlgorithmSet(as2); - - cluster2.addAlgorithmSet(as1); - cluster2.addAlgorithmSet(as3); - - } - */ - - public static void main(String[] args) throws Exception { - /* - AnsibleBridge ab = new AnsibleBridge(); - ab.printInventoryByDomainAndSets(clusters.values()); - System.out.println("-----------"); - ab.printInventoryBySets(clusters.values()); - - AnsibleWorker worker = ab.createWorker(); - - for(Algorithm a:extractAlgorithms()) { - for(Role r:ab.generateRoles(a)) { - worker.addRole(r); - } - } - - for(Dependency d:extractDependencies()) { - for(Role r:ab.generateRoles(d)) { - worker.addRole(r); - } - } - */ - - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); -// -// // create the algorithm (download it, etc etc) -// Algorithm algorithm = new Algorithm(); -// algorithm.setName("ichtyop"); -// algorithm.setClazz("org.gcube..."); -// algorithm.setDescription("some description"); -// -// Dependency d = new Dependency(); -// d.setName("libpng"); -// d.setType("os"); -// algorithm.addDependency(d); -// -// d = new Dependency(); -// d.setName("some-r-package"); -// d.setType("cran"); -// algorithm.addDependency(d); -// -// d = new Dependency(); -// d.setName("some-other-r-package"); -// d.setType("cran"); -// algorithm.addDependency(d); -// -// d = new Dependency(); -// d.setName("voodoo"); -// d.setType("custom"); -// algorithm.addDependency(d); -// -// // create the algorithm (download it, etc etc) -// Algorithm ewe = new Algorithm(); -// ewe.setName("ewe"); -// -// d = new Dependency(); -// d.setName("voodoo"); -// d.setType("custom"); -// ewe.addDependency(d); - - AlgorithmSet algorithms = new AlgorithmSet(); - algorithms.setName("dummy-set"); - - -// algorithms.addAlgorithm(algorithm); -// algorithms.addAlgorithm(ewe); - - Algorithm ensemble = new Algorithm(); - ensemble.setName("ensemble"); - ensemble.setCategory("ICHTHYOP_MODEL"); - ensemble.setAlgorithmType("transducerers"); - ensemble.setPackageURL("http://data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0"); - ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone"); - ensemble.setSkipJava("N"); - ensemble.setDescription("test"); - - Dependency d = new Dependency(); - d.setName("libpng3"); - d.setType("os"); - ensemble.addDependency(d); - algorithms.addAlgorithm(ensemble); - - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID()); - - } - -} diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java deleted file mode 100644 index 211fd04..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -import java.net.ProxySelector; - -import org.gcube.common.scope.api.ScopeProvider; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; -import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; - -public class ISClientTest { - - public static void main(String[] args) { - ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); - ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(new ISClient().listDataminersInVRE()); - } - -}