gCubeSystem
/
dataminer-pool-manager
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@142767 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-02-20 13:52:25 +00:00
parent 727756fffa
commit bab093ce03
60 changed files with 754 additions and 272 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.classpath Normal file
View File

@ -0,0 +1,34 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jst.j2ee.internal.web.container"/>
<classpathentry kind="con" path="org.eclipse.jst.j2ee.internal.module.container"/>
<classpathentry kind="lib" path="/home/ngalante/Downloads/servlet-api-2.5.jar"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>

42
.project Normal file
View File

@ -0,0 +1,42 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>dataminer-pool-manager</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.wst.jsdt.core.javascriptValidator</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.wst.common.project.facet.core.builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.wst.validation.validationbuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jem.workbench.JavaEMFNature</nature>
<nature>org.eclipse.wst.common.modulecore.ModuleCoreNature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.wst.common.project.facet.core.nature</nature>
<nature>org.eclipse.wst.jsdt.core.jsNature</nature>
</natures>
</projectDescription>

12
.settings/.jsdtscope Normal file
View File

@ -0,0 +1,12 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry excluding="**/*.min.js|**/node_modules/*|**/bower_components/*" kind="src" path="WebContent"/>
<classpathentry kind="con" path="org.eclipse.wst.jsdt.launching.JRE_CONTAINER"/>
<classpathentry kind="con" path="org.eclipse.wst.jsdt.launching.WebProject">
<attributes>
<attribute name="hide" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.wst.jsdt.launching.baseBrowserLibrary"/>
<classpathentry kind="output" path=""/>
</classpath>

5
.settings/org.eclipse.core.resources.prefs Normal file
View File

@ -0,0 +1,5 @@
eclipse.preferences.version=1
encoding***REMOVED***src/main/java=UTF-8
encoding***REMOVED***src/main/resources=UTF-8
encoding***REMOVED***src/test/java=UTF-8
encoding/<project>=UTF-8

8
.settings/org.eclipse.jdt.core.prefs Normal file
View File

@ -0,0 +1,8 @@
eclipse.preferences.version=1
org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7
org.eclipse.jdt.core.compiler.compliance=1.7
org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
org.eclipse.jdt.core.compiler.source=1.7

4
.settings/org.eclipse.m2e.core.prefs Normal file
View File

@ -0,0 +1,4 @@
activeProfiles=
eclipse.preferences.version=1
resolveWorkspaceProjects=true
version=1

10
.settings/org.eclipse.wst.common.component Normal file
View File

@ -0,0 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?><project-modules id="moduleCoreId" project-version="1.5.0">
<wb-module deploy-name="dataminer-pool-manager">
<wb-resource deploy-path="/" source-path="/WebContent" tag="defaultRootSource"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
<wb-resource deploy-path="/WEB-INF/classes" source-path="/src/test/java"/>
<property name="context-root" value="dataminer-pool-manager"/>
<property name="java-output-path" value="/dataminer-pool-manager/target/classes"/>
</wb-module>
</project-modules>

7
.settings/org.eclipse.wst.common.project.facet.core.prefs.xml Normal file
View File

@ -0,0 +1,7 @@
<root>
<facet id="jst.jaxrs">
<node name="libprov">
<attribute name="provider-id" value="jaxrs-no-op-library-provider"/>
</node>
</facet>
</root>

7
.settings/org.eclipse.wst.common.project.facet.core.xml Normal file
View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<faceted-project>
<installed facet="java" version="1.7"/>
<installed facet="jst.web" version="3.0"/>
<installed facet="wst.jsdt.web" version="1.0"/>
<installed facet="jst.jaxrs" version="2.0"/>
</faceted-project>

1
.settings/org.eclipse.wst.jsdt.ui.superType.container Normal file
View File

@ -0,0 +1 @@
org.eclipse.wst.jsdt.launching.baseBrowserLibrary

1
.settings/org.eclipse.wst.jsdt.ui.superType.name Normal file
View File

@ -0,0 +1 @@
Window

2
.settings/org.eclipse.wst.validation.prefs Normal file
View File

@ -0,0 +1,2 @@
disabled=06target
eclipse.preferences.version=1

4
0e558010-c4af-458a-b6ba-532f200d9db8 Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE log SYSTEM "logger.dtd">
<log>
</log>

3
WebContent/META-INF/MANIFEST.MF Normal file
View File

@ -0,0 +1,3 @@
Manifest-Version: 1.0
Class-Path:

2
distro/descriptor.xml
View File

@ -28,7 +28,7 @@
<files> <files>
<file> <file>
<source>target/${build.finalName***REMOVED***.jar</source> <source>target/${build.finalName***REMOVED***.war</source>
<outputDirectory>/${artifactId***REMOVED***</outputDirectory> <outputDirectory>/${artifactId***REMOVED***</outputDirectory>
</file> </file>
</files> </files>

4
f7ba9c3a-6e2c-4d90-8136-959d71b5b4d1 Normal file
View File

@ -0,0 +1,4 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE log SYSTEM "logger.dtd">
<log>
</log>

0
log Normal file
View File

32
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataanalysis</groupId> <groupId>org.gcube.dataanalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId> <artifactId>dataminer-pool-manager</artifactId>
<version>1.0.0-SNAPSHOT</version> <version>1.0.0-SNAPSHOT</version>
<packaging>jar</packaging> <packaging>war</packaging>
<name>dataminer-pool-manager</name> <name>dataminer-pool-manager</name>
<description> <description>
@ -44,6 +44,36 @@
<artifactId>ic-client</artifactId> <artifactId>ic-client</artifactId>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>2.23.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>2.22.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>2.22.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies> </dependencies>
<dependencyManagement> <dependencyManagement>

22
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/AnsibleWorker.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
View File

@ -1,14 +1,12 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible; package org.gcube.dataanalysis.dataminer.poolmanager.ansible;
import java.io.File; import java.io.File;
***REMOVED*** ***REMOVED***
import java.util.UUID;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
/** /**
* This class is responsible for the interface with ansible, retrieving log, * This class is responsible for the interface with ansible, retrieving log,
@ -45,11 +43,11 @@ public class AnsibleWorker {
this.ensureWorkStructure(); this.ensureWorkStructure();
***REMOVED*** ***REMOVED***
private File getWorkdir() { public File getWorkdir() {
return this.workerRoot; return this.workerRoot;
***REMOVED*** ***REMOVED***
private File getRolesDir() { public File getRolesDir() {
return new File(this.getWorkdir(), ROLES_DIR); return new File(this.getWorkdir(), ROLES_DIR);
***REMOVED*** ***REMOVED***
@ -57,12 +55,12 @@ public class AnsibleWorker {
return this.workerRoot.getName(); return this.workerRoot.getName();
***REMOVED*** ***REMOVED***
private void ensureWorkStructure() { public void ensureWorkStructure() {
***REMOVED*** generate root ***REMOVED*** generate root
this.getWorkdir().mkdirs(); this.getWorkdir().mkdirs();
***REMOVED*** ***REMOVED***
private void removeWorkStructure() { public void removeWorkStructure() {
***REMOVED*** remove the working dir ***REMOVED*** remove the working dir
***REMOVED*** this.getWorkdir().delete(); ***REMOVED*** this.getWorkdir().delete();
***REMOVED*** ***REMOVED***
@ -91,6 +89,8 @@ public class AnsibleWorker {
AnsibleSerializeHelper.serializeRole(r, this.getRolesDir()); AnsibleSerializeHelper.serializeRole(r, this.getRolesDir());
***REMOVED*** ***REMOVED***
public void apply() { public void apply() {
***REMOVED*** TODO execute the playbook and return output ***REMOVED*** TODO execute the playbook and return output
System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName()); System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/AnsibleHost.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
public class AnsibleHost { public class AnsibleHost {

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/HostGroup.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
import java.util.Collection; import java.util.Collection;
import java.util.Vector; import java.util.Vector;

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Inventory.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
import java.util.Collection; import java.util.Collection;
import java.util.Vector; import java.util.Vector;

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Playbook.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
***REMOVED*** ***REMOVED***
import java.util.Vector; import java.util.Vector;

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/Role.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
import java.util.Collection; import java.util.Collection;
import java.util.Vector; import java.util.Vector;

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansible/model/RoleFile.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansible.model; package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
public class RoleFile { public class RoleFile {

80
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleBridge.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -1,7 +1,9 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge;
import java.io.File; import java.io.File;
import java.io.FileOutputStream;
***REMOVED*** ***REMOVED***
import java.io.PrintStream;
import java.util.Collection; import java.util.Collection;
***REMOVED*** ***REMOVED***
import java.util.Map; import java.util.Map;
@ -11,38 +13,37 @@ import java.util.TreeSet;
import java.util.UUID; import java.util.UUID;
import java.util.Vector; import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.AnsibleHost; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.AlgorithmPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.AlgorithmPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CranDependencyPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CranDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomDependencyPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CustomDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomRoleManager;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.CustomRoleManager; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.OSDependencyPackage;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.DependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.StaticRoleManager;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.OSDependencyPackage; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.StaticRoleManager; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template.TemplateManager; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; ***REMOVED***
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster; ***REMOVED***
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host; ***REMOVED***
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.AlgorithmComparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.DependencyComparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.HostComparator;
public class AnsibleBridge { public class AnsibleBridge {
private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class);
/** /**
* The workdir for this service * The workdir for this service
*/ */
private String dpmRoot; private String dpmRoot;
public AnsibleBridge() { public AnsibleBridge() {
this("/home/paolo/tmp/dataminer-pool-manager"); this("/tmp/dataminer-pool-manager");
***REMOVED*** ***REMOVED***
public AnsibleBridge(String root) { public AnsibleBridge(String root) {
@ -142,9 +143,16 @@ public class AnsibleBridge {
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
public void applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException { public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException {
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), UUID.randomUUID().toString()));
return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString());
***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid) throws IOException {
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
List<Role> algoRoles = new Vector<>(); List<Role> algoRoles = new Vector<>();
***REMOVED*** add algorithms and dependencies to the worker ***REMOVED*** add algorithms and dependencies to the worker
@ -186,14 +194,26 @@ public class AnsibleBridge {
worker.setPlaybook(playbook); worker.setPlaybook(playbook);
***REMOVED*** execute ***REMOVED*** execute and save log locally
worker.apply(); PrintStream console = System.out;
File path = new File(worker.getWorkdir() + File.separator + "logs");
path.mkdirs();
File n = new File(path + File.separator + worker.getWorkerId());
FileOutputStream fos = new FileOutputStream(n);
PrintStream ps = new PrintStream(fos);
System.setOut(ps);
worker.apply();
System.setOut(console);
worker.apply();
System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED*** destroy the worker ***REMOVED*** destroy the worker
worker.destroy(); worker.destroy();
return worker;
***REMOVED*** ***REMOVED***
private TemplateManager getTemplateManager() { private TemplateManager getTemplateManager() {
return new TemplateManager(this.dpmRoot+"/templates"); return new TemplateManager(this.dpmRoot+"/templates");
***REMOVED*** ***REMOVED***

14
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/AnsibleSerializeHelper.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
@ -7,12 +7,12 @@ import java.io.FileNotFoundException;
import java.io.PrintWriter; import java.io.PrintWriter;
import org.apache.commons.io.IOUtils; import org.apache.commons.io.IOUtils;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.AnsibleHost; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.HostGroup; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.HostGroup;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
public class AnsibleSerializeHelper { public class AnsibleSerializeHelper {

9
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
@ -6,10 +6,9 @@ import java.util.Map;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.Vector; import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class AlgorithmPackage { public class AlgorithmPackage {

4
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CranDependencyPackage.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java
View File

@ -1,6 +1,6 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class CranDependencyPackage extends DependencyPackage { public class CranDependencyPackage extends DependencyPackage {

6
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java
View File

@ -1,11 +1,11 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.util.Collection; import java.util.Collection;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.Vector; import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class CustomDependencyPackage extends DependencyPackage { public class CustomDependencyPackage extends DependencyPackage {

11
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomRoleManager.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java
View File

@ -1,16 +1,11 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
***REMOVED*** ***REMOVED***
import java.util.Collection;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.Vector;
import org.apache.commons.io.IOUtils; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
public class CustomRoleManager { public class CustomRoleManager {

7
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
@ -6,9 +6,8 @@ import java.util.Map;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import java.util.Vector; import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class DependencyPackage { public class DependencyPackage {

4
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/OSDependencyPackage.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java
View File

@ -1,6 +1,6 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class OSDependencyPackage extends DependencyPackage { public class OSDependencyPackage extends DependencyPackage {

6
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/StaticRoleManager.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java
View File

@ -1,12 +1,12 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.io.File; import java.io.File;
***REMOVED*** ***REMOVED***
import java.util.Collection; import java.util.Collection;
import java.util.Vector; import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
public class StaticRoleManager { public class StaticRoleManager {

12
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/TemplateManager.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java
View File

@ -1,17 +1,13 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
***REMOVED*** ***REMOVED***
import java.util.HashMap;
import java.util.Map; import java.util.Map;
import java.util.NoSuchElementException; import java.util.NoSuchElementException;
import org.apache.commons.io.IOUtils; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleSerializeHelper;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
import org.stringtemplate.v4.ST; import org.stringtemplate.v4.ST;
public class TemplateManager { public class TemplateManager {

4
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/clients/ISClient.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.clients; ***REMOVED***
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
@ -9,8 +9,6 @@ import java.util.Vector;
import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.resources.gcore.ServiceEndpoint;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.gcube.resources.discovery.client.queries.api.SimpleQuery;

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Action.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
public class Action { public class Action {

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection; import java.util.Collection;
import java.util.Vector; import java.util.Vector;

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/AlgorithmSet.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection; import java.util.Collection;
import java.util.Vector; import java.util.Vector;

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Cluster.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection; import java.util.Collection;
import java.util.Vector; import java.util.Vector;

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Dependency.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
public class Dependency { public class Dependency {

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Domain.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
public class Domain { public class Domain {

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Host.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
public class Host { public class Host {

4
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/AlgorithmComparator.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java
View File

@ -1,8 +1,8 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator;
import java.util.Comparator; import java.util.Comparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
public class AlgorithmComparator implements Comparator<Algorithm> { public class AlgorithmComparator implements Comparator<Algorithm> {

5
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/DependencyComparator.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java
View File

@ -1,9 +1,8 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator;
import java.util.Comparator; import java.util.Comparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class DependencyComparator implements Comparator<Dependency> { public class DependencyComparator implements Comparator<Dependency> {

4
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/comparator/HostComparator.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java
View File

@ -1,8 +1,8 @@
package org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator;
import java.util.Comparator; import java.util.Comparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host; ***REMOVED***
public class HostComparator implements Comparator<Host> { public class HostComparator implements Comparator<Host> {

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AddAlgorithmCommand.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.process; package org.gcube.dataanalysis.dataminer.poolmanager.process;
import java.util.StringTokenizer; import java.util.StringTokenizer;

6
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AlgorithmPackageParser.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.process; package org.gcube.dataanalysis.dataminer.poolmanager.process;
***REMOVED*** ***REMOVED***
import java.io.InputStream; import java.io.InputStream;
@ -10,8 +10,8 @@ import java.util.Vector;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class AlgorithmPackageParser { public class AlgorithmPackageParser {

24
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java Normal file
View File

@ -0,0 +1,24 @@
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
***REMOVED***
import java.net.MalformedURLException;
***REMOVED***
import java.net.UnknownHostException;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
public interface PoolManager {
String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException;
Algorithm extractAlgorithm(String url) throws IOException;
String getLogById(String logId) throws IOException;
void getLogId(Algorithm algo, String vre);
String getScriptFromURL(URL logId) throws IOException;
URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException;
***REMOVED***

101
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java Normal file
View File

@ -0,0 +1,101 @@
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
***REMOVED***
import java.net.MalformedURLException;
***REMOVED***
import java.net.UnknownHostException;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
***REMOVED***
***REMOVED***
@Path("/")
public class RestPoolManager implements PoolManager {
private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class);
private PoolManager service = new DataminerPoolManager();
@GET
@Path("/add")
@Produces("text/plain")
public String addAlgorithmToVRE(@QueryParam("algorithm") String algorithm, @QueryParam("vre") String vre) throws IOException, InterruptedException {
***REMOVED*** TODO Auto-generated method stub
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
Algorithm algo = service.extractAlgorithm(algorithm);
return service.addAlgorithmToVRE(algo, vre);
***REMOVED***
@GET
@Path("/log")
@Produces("text/plain")
public String getLogById(@QueryParam("logUrl") String logUrl) throws IOException {
***REMOVED*** TODO Auto-generated method stub
LOGGER.debug("Returning Log =" + logUrl);
return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl));
***REMOVED***
@Override
public Algorithm extractAlgorithm(String url) throws IOException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
public static void main(String[] args) throws IOException, InterruptedException {
RestPoolManager a = new RestPoolManager();
a.addAlgorithmToVRE("http:***REMOVED***data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0", "/gcube/devNext/NextNext");
***REMOVED***System.out.println(a.getLogById("dadcb059-69e5-48c3-aa58-3b290ae0419d"));
***REMOVED***
@Override
public void getLogId(Algorithm algo, String vre) {
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
@Override
public String getScriptFromURL(URL logId) throws IOException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
@Override
public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
@Override
public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
***REMOVED***

265
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java Normal file
View File

@ -0,0 +1,265 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service;
***REMOVED***
import java.io.File;
***REMOVED***
***REMOVED***
import java.net.MalformedURLException;
***REMOVED***
import java.net.URLConnection;
import java.net.UnknownHostException;
import java.util.UUID;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
***REMOVED***
***REMOVED***
public class DataminerPoolManager implements PoolManager {
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
***REMOVED*** static Collection<Algorithm> algorithms;
***REMOVED***
***REMOVED*** static Collection<AlgorithmSet> sets;
***REMOVED***
***REMOVED*** static {
***REMOVED*** algorithms = new Vector<>();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public DataminerPoolManager() {
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected
***REMOVED*** * on the pool.
***REMOVED*** */
***REMOVED*** public void publishAlgorithm(Algorithm algorithm) {
***REMOVED*** algorithms.add(algorithm);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Re-deploy the given algorithm wherever it's installed
***REMOVED*** *
***REMOVED*** * @param algorithm
***REMOVED*** */
***REMOVED*** /*
***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement this ***REMOVED***
***REMOVED*** */
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add the give algorithm to the given set
***REMOVED*** *
***REMOVED*** * @param algorithmId
***REMOVED*** * @param setId
***REMOVED*** */
***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) {
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
***REMOVED*** if (set != null && algorithm != null) {
***REMOVED*** set.addAlgorithm(algorithm);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Apply the given set of algorithms to the given cluster
***REMOVED*** *
***REMOVED*** * @param setId
***REMOVED*** * @param clusterId
***REMOVED*** */
***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) {
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
***REMOVED*** if (set != null && cluster != null) {
***REMOVED*** cluster.addAlgorithmSet(set);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) {
***REMOVED*** for (AlgorithmSet set : sets) {
***REMOVED*** if (name.equals(set.getName())) {
***REMOVED*** return set;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private Algorithm getAlgorithm(String name) {
***REMOVED*** for (Algorithm a : algorithms) {
***REMOVED*** if (name.equals(a.getName())) {
***REMOVED*** return a;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) {
***REMOVED*** new Thread() {
***REMOVED*** public void run() {
***REMOVED*** while (true) {
***REMOVED*** try {
***REMOVED*** addAlgorithmToVRE(algorithm, vre);
***REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED*** ***REMOVED***log here
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***.start();
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String getLogId(){
***REMOVED*** PrintStream console = System.out;
***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
***REMOVED*** path.mkdirs();
***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
***REMOVED*** FileOutputStream fos = new FileOutputStream(n);
***REMOVED*** PrintStream ps = new PrintStream(fos);
***REMOVED*** System.setOut(ps);
***REMOVED*** worker.apply();
***REMOVED*** System.setOut(console);
***REMOVED*** worker.apply();
***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED******REMOVED***
***REMOVED*** public String getLogById(String id) throws IOException {
***REMOVED*** String strLine = null;
***REMOVED*** try{
***REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
***REMOVED*** /* read log line by line */
***REMOVED*** while ((strLine = br.readLine()) != null) {
***REMOVED*** /* parse strLine to obtain what you want */
***REMOVED*** System.out.println (strLine);
***REMOVED*** ***REMOVED***
***REMOVED*** br.close();
***REMOVED*** ***REMOVED*** catch (Exception e) {
***REMOVED*** System.err.println("Error: " + e.getMessage());
***REMOVED*** ***REMOVED***
***REMOVED*** return strLine;
***REMOVED******REMOVED***
public String getScriptFromURL(URL url) throws IOException {
if (url == null) {
return null;
***REMOVED***
URLConnection yc = url.openConnection();
BufferedReader input = new BufferedReader(new InputStreamReader(
yc.getInputStream()));
String line;
StringBuffer buffer = new StringBuffer();
while ((line = input.readLine()) != null) {
buffer.append(line + "\n");
***REMOVED***
String bufferScript = buffer.substring(0, buffer.length());
input.close();
return bufferScript;
***REMOVED***
/**
* Publish the given algorithm in the given VRE
*
* @param algorithmName
* @param vre
*
*/
public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException {
***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake");
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
Runnable r = new Runnable() {
@Override
public void run() {
***REMOVED*** TODO Auto-generated method stub
try {
addAlgorithmsToVRE(algoSet, vre, uuid);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***;
new Thread(r).start();
***REMOVED***this line will execute immediately, not waiting for your task to complete
System.out.println(uuid);
return uuid;
***REMOVED***
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException{
File path = new File("/tmp/dataminer-pool-manager/work/" + File.separator + a+File.separator +"logs");
path.mkdirs();
File n = new File(path + File.separator +a);
***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress();
return new File(n.getPath()).toURI().toURL();
***REMOVED***
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException {
***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
for(Host h:new ISClient().listDataminersInVRE()) {
cluster.addHost(h);
***REMOVED***
***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId();
***REMOVED***
public Algorithm extractAlgorithm(String url) throws IOException {
return new AlgorithmPackageParser().parsePackage(url);
***REMOVED***
@Override
public void getLogId(Algorithm algo, String vre) {
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
@Override
public String getLogById(String logId) throws IOException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
***REMOVED***

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/util/PropertiesBasedProxySelector.java → src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java
View File

@ -1,4 +1,4 @@
package org.gcube.dataanalysys.dataminerpoolmanager.util; package org.gcube.dataanalysis.dataminer.poolmanager.util;
***REMOVED*** ***REMOVED***
import java.net.Authenticator; import java.net.Authenticator;

122
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/service/DataminerPoolManager.java
View File

@ -1,122 +0,0 @@
package org.gcube.dataanalysys.dataminerpoolmanager.service;
***REMOVED***
import java.util.Collection;
import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysys.dataminerpoolmanager.clients.ISClient;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
public class DataminerPoolManager {
***REMOVED*** static Collection<Algorithm> algorithms;
***REMOVED***
***REMOVED*** static Collection<AlgorithmSet> sets;
***REMOVED***
***REMOVED*** static {
***REMOVED*** algorithms = new Vector<>();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public DataminerPoolManager() {
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected
***REMOVED*** * on the pool.
***REMOVED*** */
***REMOVED*** public void publishAlgorithm(Algorithm algorithm) {
***REMOVED*** algorithms.add(algorithm);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Re-deploy the given algorithm wherever it's installed
***REMOVED*** *
***REMOVED*** * @param algorithm
***REMOVED*** */
***REMOVED*** /*
***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) { ***REMOVED*** TODO implement this ***REMOVED***
***REMOVED*** */
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add the give algorithm to the given set
***REMOVED*** *
***REMOVED*** * @param algorithmId
***REMOVED*** * @param setId
***REMOVED*** */
***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) {
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
***REMOVED*** if (set != null && algorithm != null) {
***REMOVED*** set.addAlgorithm(algorithm);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Apply the given set of algorithms to the given cluster
***REMOVED*** *
***REMOVED*** * @param setId
***REMOVED*** * @param clusterId
***REMOVED*** */
***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) {
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
***REMOVED*** if (set != null && cluster != null) {
***REMOVED*** cluster.addAlgorithmSet(set);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) {
***REMOVED*** for (AlgorithmSet set : sets) {
***REMOVED*** if (name.equals(set.getName())) {
***REMOVED*** return set;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private Algorithm getAlgorithm(String name) {
***REMOVED*** for (Algorithm a : algorithms) {
***REMOVED*** if (name.equals(a.getName())) {
***REMOVED*** return a;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
/**
* Publish the given algorithm in the given VRE
*
* @param algorithmName
* @param vre
*/
public void addAlgorithmToVRE(Algorithm algorithm, String vre) throws IOException {
***REMOVED*** create a fake algorithm set
AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake");
algoSet.addAlgorithm(algorithm);
this.addAlgorithmsToVRE(algoSet, vre);
***REMOVED***
public void addAlgorithmsToVRE(AlgorithmSet algorithms, String vre) throws IOException {
***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
for(Host h:new ISClient().listDataminersInVRE()) {
cluster.addHost(h);
***REMOVED***
***REMOVED*** apply the changes
new AnsibleBridge().applyAlgorithmSetToCluster(algorithms, cluster);
***REMOVED***
***REMOVED***

13
src/main/webapp/WEB-INF/gcube-app.xml Normal file
View File

@ -0,0 +1,13 @@
<application mode="online">
<name>dataminer-pool-manager</name>
<group>dataanalysis</group>
<version>0.0.1</version>
<!--<description>Lorem ipsum dolor sit amet...</description>-->
<!--<persistence location="/some/custom/location</persistence" />-->
<!-- <exclude>*</exclude> -->
</application>

11
src/main/webapp/WEB-INF/gcube-handlers.xml Normal file
View File

@ -0,0 +1,11 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<handlers>
<lifecycle>
<profile-management />
<!-- <plugin-registration-handler /> -->
</lifecycle>
<request>
<request-validation />
<request-accounting />
</request>
</handlers>

24
src/main/webapp/WEB-INF/web.xml Normal file
View File

@ -0,0 +1,24 @@
<!DOCTYPE web-app PUBLIC
"-***REMOVED***Sun Microsystems, Inc.***REMOVED***DTD Web Application 2.3***REMOVED***EN"
"http:***REMOVED***java.sun.com/dtd/web-app_2_3.dtd" >
<web-app>
<display-name>Archetype Created Web Application</display-name>
<servlet>
<servlet-name>REST-API</servlet-name>
<servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>jersey.config.server.provider.packages</param-name>
<param-value>org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>REST-API</servlet-name>
<url-pattern>/rest/*</url-pattern>
</servlet-mapping>
</web-app>

18
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java
View File

@ -7,12 +7,11 @@ import java.net.ProxySelector;
***REMOVED*** ***REMOVED***
import java.util.Vector; import java.util.Vector;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysys.dataminerpoolmanager.process.AddAlgorithmCommand; import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand;
import org.gcube.dataanalysys.dataminerpoolmanager.process.AlgorithmPackageParser; import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
import org.gcube.dataanalysys.dataminerpoolmanager.service.DataminerPoolManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
import org.gcube.dataanalysys.dataminerpoolmanager.util.PropertiesBasedProxySelector;
public class AlgorithmPackageParserTest { public class AlgorithmPackageParserTest {
@ -48,8 +47,9 @@ public class AlgorithmPackageParserTest {
algorithms.addAlgorithm(algo); algorithms.addAlgorithm(algo);
break; break;
***REMOVED*** ***REMOVED***
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, ***REMOVED***to uncomment
"/gcube/devNext/NextNext"); ***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms,
***REMOVED*** "/gcube/devNext/NextNext");
***REMOVED*** ***REMOVED***
/** /**
@ -111,7 +111,7 @@ public class AlgorithmPackageParserTest {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
ProxySelector.setDefault(new PropertiesBasedProxySelector( ProxySelector.setDefault(new PropertiesBasedProxySelector(
"/home/paolo/.proxy-settings")); "/home/ngalante/.proxy-settings"));
new AlgorithmPackageParserTest().extractAllAlgorithms(); new AlgorithmPackageParserTest().extractAllAlgorithms();
***REMOVED*** ***REMOVED***

10
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java
View File

@ -4,11 +4,11 @@ import java.io.File;
***REMOVED*** ***REMOVED***
import java.util.UUID; import java.util.UUID;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
public class AnsibleWorkerTest { public class AnsibleWorkerTest {

26
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java
View File

@ -1,26 +1,12 @@
package org.gcube.dataanalysis.dataminerpoolmanager; package org.gcube.dataanalysis.dataminerpoolmanager;
import java.net.ProxySelector; import java.net.ProxySelector;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeSet;
import java.util.Vector;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Cluster;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Domain;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Host;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.AlgorithmComparator;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.comparator.DependencyComparator;
import org.gcube.dataanalysys.dataminerpoolmanager.service.DataminerPoolManager;
import org.gcube.dataanalysys.dataminerpoolmanager.util.PropertiesBasedProxySelector;
public class DataminerPoolManagerTest { public class DataminerPoolManagerTest {
@ -173,7 +159,7 @@ public class DataminerPoolManagerTest {
ScopeProvider.instance.set("/gcube/devNext/NextNext"); ScopeProvider.instance.set("/gcube/devNext/NextNext");
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/paolo/.proxy-settings")); ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
***REMOVED*** create the algorithm (download it, etc etc) ***REMOVED*** create the algorithm (download it, etc etc)
Algorithm algorithm = new Algorithm(); Algorithm algorithm = new Algorithm();
@ -225,7 +211,7 @@ public class DataminerPoolManagerTest {
ensemble.addDependency(d); ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble); algorithms.addAlgorithm(ensemble);
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext"); ***REMOVED***new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext");
***REMOVED*** ***REMOVED***

6
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java
View File

@ -3,13 +3,13 @@ package org.gcube.dataanalysis.dataminerpoolmanager;
import java.net.ProxySelector; import java.net.ProxySelector;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysys.dataminerpoolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysys.dataminerpoolmanager.util.PropertiesBasedProxySelector; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class ISClientTest { public class ISClientTest {
public static void main(String[] args) { public static void main(String[] args) {
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/paolo/.proxy-settings")); ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
ScopeProvider.instance.set("/gcube/devNext/NextNext"); ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(new ISClient().listDataminersInVRE()); System.out.println(new ISClient().listDataminersInVRE());
***REMOVED*** ***REMOVED***