gCubeSystem
/
dataminer-pool-manager
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Share project "dataminer-pool-manager" into "http://svn.research-infrastructures.eu/d4science/gcube" 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@144668 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante 2017-03-03 18:07:54 +00:00
parent f307318f28
commit 00f5f0b756
51 changed files with 0 additions and 3606 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
.classpath
View File

@ -1,34 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" output="target/classes" path="src/main/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="src" output="target/test-classes" path="src/test/java">
<attributes>
<attribute name="optional" value="true"/>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.7">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
<attributes>
<attribute name="maven.pomderived" value="true"/>
</attributes>
</classpathentry>
<classpathentry kind="con" path="org.eclipse.jst.j2ee.internal.web.container"/>
<classpathentry kind="con" path="org.eclipse.jst.j2ee.internal.module.container"/>
<classpathentry kind="lib" path="/home/ngalante/Downloads/servlet-api-2.5.jar"/>
<classpathentry kind="output" path="target/classes"/>
</classpath>

42
.project
View File

@ -1,42 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>dataminer-pool-manager</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.wst.jsdt.core.javascriptValidator</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.wst.common.project.facet.core.builder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.wst.validation.validationbuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.m2e.core.maven2Builder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jem.workbench.JavaEMFNature</nature>
<nature>org.eclipse.wst.common.modulecore.ModuleCoreNature</nature>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.eclipse.m2e.core.maven2Nature</nature>
<nature>org.eclipse.wst.common.project.facet.core.nature</nature>
<nature>org.eclipse.wst.jsdt.core.jsNature</nature>
</natures>
</projectDescription>

1
distro/LICENSE
View File

@ -1 +0,0 @@
$***REMOVED***gcube.license***REMOVED***

60
distro/README
View File

@ -1,60 +0,0 @@
The gCube System - $***REMOVED***name***REMOVED***
--------------------------------------------------
$***REMOVED***description***REMOVED***
$***REMOVED***gcube.description***REMOVED***
$***REMOVED***gcube.funding***REMOVED***
Version
--------------------------------------------------
$***REMOVED***version***REMOVED*** ($***REMOVED***buildDate***REMOVED***)
Please see the file named "changelog.xml" in this directory for the release notes.
Authors
--------------------------------------------------
* Paolo Fabriani (paolo.fabriani-AT-eng.it)
Engineering Ingegneria Informatica S.p.A., Italy
Maintainers
--------------------------------------------------
* Paolo Fabriani (paolo.fabriani-AT-eng.it)
Engineering Ingegneria Informatica S.p.A., Italy
Download information
--------------------------------------------------
Source code is available from SVN:
$***REMOVED***scm.url***REMOVED***
Binaries can be downloaded from the gCube website:
$***REMOVED***gcube.website***REMOVED***
Installation
--------------------------------------------------
Installation documentation is available on-line in the gCube Wiki:
$***REMOVED***gcube.wikiRoot***REMOVED***/[admin guide page name here]
Documentation
--------------------------------------------------
Documentation is available on-line in the gCube Wiki:
$***REMOVED***gcube.wikiRoot***REMOVED***[user guide page name here]
$***REMOVED***gcube.wikiRoot***REMOVED***[developer guide page name here]
Support
--------------------------------------------------
Bugs and support requests can be reported in the gCube issue tracking tool:
$***REMOVED***gcube.issueTracking***REMOVED***
Licensing
--------------------------------------------------
This software is licensed under the terms you may find in the file named "LICENSE" in this directory.

36
distro/descriptor.xml
View File

@ -1,36 +0,0 @@
<assembly
xmlns="http:***REMOVED***maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
xmlns:xsi="http:***REMOVED***www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http:***REMOVED***maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http:***REMOVED***maven.apache.org/xsd/assembly-1.1.0.xsd">
<id>servicearchive</id>
<formats>
<format>tar.gz</format>
</formats>
<baseDirectory>/</baseDirectory>
<fileSets>
<fileSet>
<directory>$***REMOVED***distroDirectory***REMOVED***</directory>
<outputDirectory>/</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
<includes>
<include>README</include>
<include>LICENSE</include>
<include>profile.xml</include>
</includes>
<fileMode>755</fileMode>
<filtered>true</filtered>
</fileSet>
</fileSets>
<files>
<file>
<source>target/$***REMOVED***build.finalName***REMOVED***.war</source>
<outputDirectory>/$***REMOVED***artifactId***REMOVED***</outputDirectory>
</file>
</files>
</assembly>

25
distro/profile.xml
View File

@ -1,25 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<Resource xmlns:xsi="http:***REMOVED***www.w3.org/2001/XMLSchema-instance">
<ID />
<Type>Library</Type>
<Profile>
<Description>$***REMOVED***description***REMOVED***</Description>
<Class>DataminerPoolManager</Class>
<Name>$***REMOVED***artifactId***REMOVED***</Name>
<Version>1.0.0</Version>
<Packages>
<Software>
<Name>$***REMOVED***artifactId***REMOVED***</Name>
<Version>$***REMOVED***version***REMOVED***</Version>
<MavenCoordinates>
<groupId>$***REMOVED***groupId***REMOVED***</groupId>
<artifactId>$***REMOVED***artifactId***REMOVED***</artifactId>
<version>$***REMOVED***version***REMOVED***</version>
</MavenCoordinates>
<Files>
<File>$***REMOVED***build.finalName***REMOVED***.jar</File>
</Files>
</Software>
</Packages>
</Profile>
</Resource>

125
pom.xml
View File

@ -1,125 +0,0 @@
<project xmlns="http:***REMOVED***maven.apache.org/POM/4.0.0" xmlns:xsi="http:***REMOVED***www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>maven-parent</artifactId>
<groupId>org.gcube.tools</groupId>
<version>1.0.0</version>
<relativePath />
</parent>
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<version>1.0.0-SNAPSHOT</version>
<packaging>war</packaging>
<name>dataminer-pool-manager</name>
<description>
</description>
<properties>
<distroDirectory>distro</distroDirectory>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<dependencies>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>[2.5.0,2.6.0)</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>stringtemplate</artifactId>
<version>[4.0.0, 4.1.0)</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
<artifactId>ic-client</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>2.23.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>2.22.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>2.22.1</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-smartgears-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.2</version>
<configuration>
<descriptors>
<descriptor>$***REMOVED***distroDirectory***REMOVED***/descriptor.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>servicearchive</id>
<phase>install</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

135
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
View File

@ -1,135 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible;
import java.io.BufferedReader;
import java.io.File;
***REMOVED***
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.util.Scanner;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
/**
* This class is responsible for the interface with ansible, retrieving log,
* etc. etc. It's not supposed to access templates and static stuff files. It
* does not know the service datamodel.
*
* @author paolo
*
*/
public class AnsibleWorker ***REMOVED***
/**
* The name of the inventory
*/
private static String INVENTORY_NAME = "inventory.yaml";
/**
* The directory containing roles
*/
private static String ROLES_DIR = "roles";
/**
* The name of the playbook
*/
private static String PLAYBOOK_NAME = "playbook.yaml";
/**
* The root of the worker. This corresponds to a standard ansible working dir.
*/
private File workerRoot;
public AnsibleWorker(File root) ***REMOVED***
this.workerRoot = root;
this.ensureWorkStructure();
***REMOVED***
public File getWorkdir() ***REMOVED***
return this.workerRoot;
***REMOVED***
public File getRolesDir() ***REMOVED***
return new File(this.getWorkdir(), ROLES_DIR);
***REMOVED***
public String getWorkerId() ***REMOVED***
return this.workerRoot.getName();
***REMOVED***
public void ensureWorkStructure() ***REMOVED***
***REMOVED*** generate root
this.getWorkdir().mkdirs();
***REMOVED***
public void removeWorkStructure() ***REMOVED***
***REMOVED*** remove the working dir
this.getWorkdir().delete();
***REMOVED***
public File getPlaybookFile() ***REMOVED***
return new File(this.getWorkdir(), PLAYBOOK_NAME);
***REMOVED***
public File getInventoryFile() ***REMOVED***
return new File(this.getWorkdir(), INVENTORY_NAME);
***REMOVED***
public void setInventory(Inventory inventory) throws IOException ***REMOVED***
***REMOVED*** serialize the string to the 'inventory' file
AnsibleSerializeHelper.serialize(inventory, this.getInventoryFile());
***REMOVED***
public void setPlaybook(Playbook playbook) throws IOException ***REMOVED***
***REMOVED*** serialize the string to the 'playbook' file
AnsibleSerializeHelper.serialize(playbook, this.getPlaybookFile());
***REMOVED***
public void addRole(Role r) throws IOException ***REMOVED***
***REMOVED*** Serialize role in the workdir
AnsibleSerializeHelper.serializeRole(r, this.getRolesDir());
***REMOVED***
public void apply() throws IOException ***REMOVED***
***REMOVED*** TODO execute the playbook and return output
System.out.println(this.getWorkdir());
try ***REMOVED***
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() + " " + this.getPlaybookFile().getAbsolutePath());
inheritIO(p.getInputStream(), System.out);
inheritIO(p.getErrorStream(), System.err);
***REMOVED*** catch (IOException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***System.out.println("TODO: execute: ansible-playbook -v -i " + this.getInventoryFile().getName() + " " + this.getPlaybookFile().getName());
***REMOVED***
private static void inheritIO(final InputStream src, final PrintStream dest) ***REMOVED***
new Thread(new Runnable() ***REMOVED***
public void run() ***REMOVED***
Scanner sc = new Scanner(src);
while (sc.hasNextLine()) ***REMOVED***
dest.println(sc.nextLine());
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED***
/**
* Destroy the worker:
* - remove the working dir
*/
public void destroy() ***REMOVED***
this.removeWorkStructure();
***REMOVED***
***REMOVED***

19
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/AnsibleHost.java
View File

@ -1,19 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
public class AnsibleHost ***REMOVED***
private String name;
public AnsibleHost(String name) ***REMOVED***
this.name = name;
***REMOVED***
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
***REMOVED***

29
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/HostGroup.java
View File

@ -1,29 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
import java.util.Collection;
import java.util.Vector;
public class HostGroup ***REMOVED***
private String name;
private Collection<AnsibleHost> hosts;
public HostGroup(String name) ***REMOVED***
this.name = name;
this.hosts = new Vector<>();
***REMOVED***
public void addHost(AnsibleHost h) ***REMOVED***
this.hosts.add(h);
***REMOVED***
public String getName() ***REMOVED***
return this.name;
***REMOVED***
public Collection<AnsibleHost> getHosts() ***REMOVED***
return new Vector<>(this.hosts);
***REMOVED***
***REMOVED***

37
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Inventory.java
View File

@ -1,37 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
import java.util.Collection;
import java.util.Vector;
public class Inventory ***REMOVED***
private Collection<HostGroup> groups;
public Inventory() ***REMOVED***
this.groups = new Vector<>();
***REMOVED***
public void addGroup(HostGroup group) ***REMOVED***
this.groups.add(group);
***REMOVED***
public void addHost(AnsibleHost h, String groupName) ***REMOVED***
this.getGroup(groupName).addHost(h);
***REMOVED***
private HostGroup getGroup(String groupName) ***REMOVED***
for (HostGroup hg : this.groups) ***REMOVED***
if (groupName.equals(hg.getName())) ***REMOVED***
return hg;
***REMOVED***
***REMOVED***
HostGroup hg = new HostGroup(groupName);
this.groups.add(hg);
return hg;
***REMOVED***
public Collection<HostGroup> getHostGroups() ***REMOVED***
return new Vector<>(this.groups);
***REMOVED***
***REMOVED***

50
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Playbook.java
View File

@ -1,50 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
***REMOVED***
import java.util.Vector;
public class Playbook ***REMOVED***
private String hostGroupName;
private List<String> roles;
private String remote_user;
public Playbook() ***REMOVED***
this.roles = new Vector<>();
***REMOVED***
public void addRole(String role) ***REMOVED***
roles.add(role);
***REMOVED***
public void applyTo(String hostGroupName) ***REMOVED***
this.hostGroupName = hostGroupName;
***REMOVED***
public String getHostGroupName() ***REMOVED***
return hostGroupName;
***REMOVED***
public List<String> getRoles() ***REMOVED***
return new Vector<>(roles);
***REMOVED***
public String getRemote_user() ***REMOVED***
return remote_user;
***REMOVED***
public void setRemote_user(String remote_user) ***REMOVED***
this.remote_user = remote_user;
***REMOVED***
public void setHostGroupName(String hostGroupName) ***REMOVED***
this.hostGroupName = hostGroupName;
***REMOVED***
public void setRoles(List<String> roles) ***REMOVED***
this.roles = roles;
***REMOVED***
***REMOVED***

51
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/Role.java
View File

@ -1,51 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
import java.util.Collection;
import java.util.Vector;
public class Role ***REMOVED***
/**
* The name of the role
*/
private String name;
private Collection<RoleFile> tasks;
private Collection<RoleFile> meta;
public Role() ***REMOVED***
this.tasks = new Vector<>();
this.meta = new Vector<>();
***REMOVED***
public Role(String name) ***REMOVED***
this();
this.name = name;
***REMOVED***
public void addTaskFile(RoleFile tf) ***REMOVED***
this.tasks.add(tf);
***REMOVED***
public void addMeta(RoleFile tf) ***REMOVED***
this.meta.add(tf);
***REMOVED***
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
public Collection<RoleFile> getTaskFiles() ***REMOVED***
return new Vector<>(this.tasks);
***REMOVED***
public Collection<RoleFile> getMeta() ***REMOVED***
return new Vector<>(this.meta);
***REMOVED***
***REMOVED***

54
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/model/RoleFile.java
View File

@ -1,54 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansible.model;
public class RoleFile ***REMOVED***
/**
* The path to the file, starting from the role root
*/
private String path;
/**
* The name of the task file
*/
private String name;
/**
* The content of the task file
* @return
*/
private String content;
public RoleFile() ***REMOVED***
***REMOVED***
public RoleFile(String name, String content) ***REMOVED***
this();
this.setName(name);
this.setContent(content);
***REMOVED***
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
public String getContent() ***REMOVED***
return content;
***REMOVED***
public void setContent(String content) ***REMOVED***
this.content = content;
***REMOVED***
public String getPath() ***REMOVED***
return path;
***REMOVED***
public void setPath(String path) ***REMOVED***
this.path = path;
***REMOVED***
***REMOVED***

276
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -1,276 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge;
import java.io.File;
import java.io.FileOutputStream;
***REMOVED***
import java.io.PrintStream;
import java.util.Collection;
***REMOVED***
import java.util.Map;
***REMOVED***
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
import java.util.Vector;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.AlgorithmPackage;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CranDependencyPackage;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomDependencyPackage;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.CustomRoleManager;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.OSDependencyPackage;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.StaticRoleManager;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template.TemplateManager;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
***REMOVED***
public class AnsibleBridge ***REMOVED***
private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class);
/**
* The workdir for this service
*/
private String dpmRoot;
public AnsibleBridge() ***REMOVED***
this(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
***REMOVED***
public AnsibleBridge(String root) ***REMOVED***
this.dpmRoot = root;
this.ensureServiceRoot();
***REMOVED***
private void ensureServiceRoot() ***REMOVED***
***REMOVED*** generate root
new File(dpmRoot).mkdirs();
***REMOVED*** 'template' is for template roles
this.getTemplatesDir().mkdirs();
***REMOVED*** 'static' is for custom roles
this.getCustomDir().mkdirs();
***REMOVED*** 'work' is for temporary working directories
this.getWorkDir().mkdirs();
***REMOVED***
private File getWorkDir() ***REMOVED***
return new File(this.dpmRoot, "work");
***REMOVED***
private File getTemplatesDir() ***REMOVED***
return new File(this.dpmRoot, "templates");
***REMOVED***
private File getCustomDir() ***REMOVED***
return new File(this.dpmRoot, "custom");
***REMOVED***
public AnsibleWorker createWorker() ***REMOVED***
File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString());
AnsibleWorker worker = new AnsibleWorker(workerRoot);
return worker;
***REMOVED***
/**
* Groups hosts by domain and algorithm sets
* @param clusters
*/
public void printInventoryByDomainAndSets(Collection<Cluster> clusters) ***REMOVED***
Map<String, Set<Host>> inventory = new TreeMap<>();
for(Cluster cluster:clusters) ***REMOVED***
for(AlgorithmSet as:cluster.getAlgorithmSets()) ***REMOVED***
String asName = as.getName();
for(Host h:cluster.getHosts()) ***REMOVED***
String domain = h.getDomain().getName();
String key = String.format("[%s@%s]", asName, domain);
Set<Host> hosts = inventory.get(key);
if(hosts==null) ***REMOVED***
hosts = new TreeSet<>(new HostComparator());
inventory.put(key, hosts);
***REMOVED***
hosts.add(h);
***REMOVED***
***REMOVED***
***REMOVED***
for(String key:inventory.keySet()) ***REMOVED***
System.out.println(key);
Collection<Host> hosts = inventory.get(key);
for(Host h:hosts) ***REMOVED***
System.out.println(h.getName()+"."+h.getDomain().getName());
***REMOVED***
System.out.println();
***REMOVED***
***REMOVED***
/**
* Groups hosts by algorithm sets only
* @param clusters
*/
public void printInventoryBySets(Collection<Cluster> clusters) ***REMOVED***
Map<String, Set<Host>> inventory = new TreeMap<>();
for (Cluster cluster : clusters) ***REMOVED***
for (AlgorithmSet as : cluster.getAlgorithmSets()) ***REMOVED***
String asName = as.getName();
for (Host h : cluster.getHosts()) ***REMOVED***
String key = String.format("[%s]", asName);
Set<Host> hosts = inventory.get(key);
if (hosts == null) ***REMOVED***
hosts = new TreeSet<>(new HostComparator());
inventory.put(key, hosts);
***REMOVED***
hosts.add(h);
***REMOVED***
***REMOVED***
***REMOVED***
for (String key : inventory.keySet()) ***REMOVED***
System.out.println(key);
Collection<Host> hosts = inventory.get(key);
for (Host h : hosts) ***REMOVED***
System.out.println(h.getName()+"."+h.getDomain().getName());
***REMOVED***
System.out.println();
***REMOVED***
***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException ***REMOVED***
return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString());
***REMOVED***
public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid) throws IOException ***REMOVED***
AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid));
List<Role> algoRoles = new Vector<>();
***REMOVED*** add algorithms and dependencies to the worker
for (Algorithm a : as.getAlgorithms()) ***REMOVED***
for (Role r : this.generateRoles(a)) ***REMOVED***
algoRoles.add(r);
worker.addRole(r);
***REMOVED***
for (Dependency d : a.getDependencies()) ***REMOVED***
for (Role r : this.generateRoles(d)) ***REMOVED***
worker.addRole(r);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** add static roles
for(Role r:this.getStaticRoleManager().getStaticRoles()) ***REMOVED***
worker.addRole(r);
***REMOVED***
***REMOVED*** generate the inventory
Inventory inventory = new Inventory();
for (Host h : cluster.getHosts()) ***REMOVED***
AnsibleHost ah = new AnsibleHost(h.getName());
inventory.addHost(ah, "universe");
inventory.addHost(ah, "d4science");
***REMOVED***
worker.setInventory(inventory);
***REMOVED*** generate the playbook
Playbook playbook = new Playbook();
playbook.setRemote_user("root");
playbook.applyTo("universe");
for(Role r:algoRoles) ***REMOVED***
***REMOVED*** add only 'add' roles
if(!r.getName().endsWith("remove")) ***REMOVED***
playbook.addRole(r.getName());
***REMOVED***
***REMOVED***
worker.setPlaybook(playbook);
***REMOVED*** execute and save log locally
PrintStream console = System.out;
File path = new File(worker.getWorkdir() + File.separator + "logs");
path.mkdirs();
File n = new File(path + File.separator + worker.getWorkerId());
FileOutputStream fos = new FileOutputStream(n);
PrintStream ps = new PrintStream(fos);
System.setOut(ps);
System.setErr(ps);
***REMOVED***System.setErr(console);
worker.apply();
***REMOVED***System.setOut(console);
***REMOVED***worker.apply();
System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED*** destroy the worker
worker.destroy();
return worker;
***REMOVED***
private TemplateManager getTemplateManager() ***REMOVED***
return new TemplateManager(this.dpmRoot+"/templates");
***REMOVED***
private CustomRoleManager getCustomRoleManager() ***REMOVED***
return new CustomRoleManager(this.dpmRoot+"/custom");
***REMOVED***
private StaticRoleManager getStaticRoleManager() ***REMOVED***
return new StaticRoleManager(this.dpmRoot+"/static");
***REMOVED***
/**
* Generate all roles for this dependency
* @param d
*/
public Collection<Role> generateRoles(Dependency d) ***REMOVED***
Collection<Role> roles = new Vector<>();
if("os".equalsIgnoreCase(d.getType())) ***REMOVED***
OSDependencyPackage pkg = new OSDependencyPackage(d);
if(pkg!=null) ***REMOVED***
roles.addAll(pkg.getRoles(this.getTemplateManager()));
***REMOVED***
***REMOVED*** else if("custom".equalsIgnoreCase(d.getType())) ***REMOVED***
CustomDependencyPackage pkg = new CustomDependencyPackage(d);
if(pkg!=null) ***REMOVED***
roles.addAll(pkg.getRoles(this.getCustomRoleManager()));
***REMOVED***
***REMOVED***
else if("github".equalsIgnoreCase(d.getType())) ***REMOVED***
CranDependencyPackage pkg = new CranDependencyPackage(d);
if(pkg!=null) ***REMOVED***
roles.addAll(pkg.getRoles(this.getTemplateManager()));
***REMOVED***
***REMOVED***
else if("cran".equalsIgnoreCase(d.getType())) ***REMOVED***
CranDependencyPackage pkg = new CranDependencyPackage(d);
if(pkg!=null) ***REMOVED***
roles.addAll(pkg.getRoles(this.getTemplateManager()));
***REMOVED***
***REMOVED***
return roles;
***REMOVED***
public Collection<Role> generateRoles(Algorithm a) ***REMOVED***
AlgorithmPackage pkg = new AlgorithmPackage(a);
return pkg.getRoles(this.getTemplateManager());
***REMOVED***
***REMOVED***

119
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleSerializeHelper.java
View File

@ -1,119 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
***REMOVED***
import java.io.PrintWriter;
import org.apache.commons.io.IOUtils;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.AnsibleHost;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.HostGroup;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
public class AnsibleSerializeHelper ***REMOVED***
public static void serialize(Inventory inventory, File inventoryFile) throws IOException ***REMOVED***
String out = "";
for(HostGroup hg:inventory.getHostGroups()) ***REMOVED***
out+=String.format("[%s]\n", hg.getName());
for(AnsibleHost h:hg.getHosts()) ***REMOVED***
out+=h.getName()+"\n";
***REMOVED***
out+="\n";
***REMOVED***
out = out.trim();
serialize(out, inventoryFile);
***REMOVED***
public static void serialize(Playbook playbook, File playbookFile) throws IOException ***REMOVED***
String out = "- hosts: " + playbook.getHostGroupName() + "\n";
out += " remote_user: "+playbook.getRemote_user()+"\n";
out+=" roles:\n";
for(String r:playbook.getRoles()) ***REMOVED***
out+=" - " + r+"\n";
***REMOVED***
out+=" vars:\n";
out+=" os_package_state: present\n";
out = out.trim();
serialize(out, playbookFile);
***REMOVED***
public static void serializeRole(Role r, File dir) throws IOException ***REMOVED***
***REMOVED*** create root
File root = new File(dir, r.getName());
root.mkdirs();
***REMOVED*** create tasks
if(r.getTaskFiles().size()>0) ***REMOVED***
File tasks = new File(root, "tasks");
tasks.mkdirs();
for(RoleFile tf: r.getTaskFiles()) ***REMOVED***
serializeTask(tf, tasks);
***REMOVED***
***REMOVED***
***REMOVED*** create meta
if(r.getMeta().size()>0) ***REMOVED***
File meta = new File(root, "meta");
meta.mkdirs();
for(RoleFile tf: r.getMeta()) ***REMOVED***
serializeTask(tf, meta);
***REMOVED***
***REMOVED***
***REMOVED***
public static void serializeTask(RoleFile tf, File dir) throws IOException ***REMOVED***
File f = new File(dir, tf.getName());
serialize(tf.getContent().trim(), f);
***REMOVED***
public static void serialize(String s, File f) throws IOException ***REMOVED***
PrintWriter out = new PrintWriter(f);
out.println(s);
out.close();
***REMOVED***
public static Role deserializeRoleFromFilesystem(File roleDir) throws IOException ***REMOVED***
Role out = new Role();
out.setName(roleDir.getName());
if(!roleDir.exists()) ***REMOVED***
throw new FileNotFoundException();
***REMOVED***
try ***REMOVED***
File tasksDir = new File(roleDir, "tasks");
if(tasksDir.exists()) ***REMOVED***
for(File main:tasksDir.listFiles()) ***REMOVED***
String content = IOUtils.toString(new FileInputStream(main), "UTF-8");
RoleFile tf = new RoleFile(main.getName(), content);
tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1));
out.addTaskFile(tf);
***REMOVED***
***REMOVED***
***REMOVED*** catch(FileNotFoundException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
try ***REMOVED***
File metaDir = new File(roleDir, "meta");
if(metaDir.exists()) ***REMOVED***
for(File main:metaDir.listFiles()) ***REMOVED***
String content = IOUtils.toString(new FileInputStream(main), "UTF-8");
RoleFile tf = new RoleFile(main.getName(), content);
tf.setPath(main.getAbsolutePath().substring(roleDir.getAbsolutePath().length()+1));
out.addMeta(tf);
***REMOVED***
***REMOVED***
***REMOVED*** catch(FileNotFoundException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
return out;
***REMOVED***
***REMOVED***

71
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/AlgorithmPackage.java
View File

@ -1,71 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Vector;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class AlgorithmPackage ***REMOVED***
private Algorithm algorithm;
public AlgorithmPackage(Algorithm a) ***REMOVED***
this.algorithm = a;
***REMOVED***
protected Map<String, String> getDictionary(Algorithm a) ***REMOVED***
Map<String, String> out = new HashMap<String, String>();
out.put("name", a.getName());
out.put("category", a.getCategory());
out.put("class", a.getClazz());
out.put("atype", a.getAlgorithmType());
out.put("skipjava", a.getSkipJava());
out.put("vre", ScopeProvider.instance.get());
***REMOVED***out.put("vre", "FAKE_VRE");
out.put("packageurl", a.getPackageURL());
out.put("description", a.getDescription());
String deps = "";
for(Dependency d:a.getDependencies()) ***REMOVED***
deps+=String.format("- ***REMOVED*** role: %s ***REMOVED***\n", d.getType()+"-"+d.getName().replaceAll("/", "-"));
***REMOVED***
deps = deps.trim();
out.put("dependencies", deps);
return out;
***REMOVED***
protected Algorithm getAlgorithm() ***REMOVED***
return this.algorithm;
***REMOVED***
public Collection<Role> getRoles(TemplateManager tm) ***REMOVED***
Collection<Role> out = new Vector<>();
for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update"
String roleName = "gcube-algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode);
try ***REMOVED***
***REMOVED*** find template
Role template = tm.getRoleTemplate("gcube-algorithm-" + mode);
***REMOVED***
if(template!=null) ***REMOVED***
Map<String, String> dictionary = this.getDictionary(this.getAlgorithm());
Role r = tm.fillRoleTemplate(template, dictionary);
r.setName(roleName);
out.add(r);
***REMOVED*** else ***REMOVED***
System.out.println("WARNING: template is null");
***REMOVED***
***REMOVED*** catch (NoSuchElementException e) ***REMOVED***
***REMOVED*** e.printStackTrace();
System.out.println("WARNING: no template found for " + roleName);
***REMOVED***
***REMOVED***
return out;
***REMOVED***
***REMOVED***

11
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CranDependencyPackage.java
View File

@ -1,11 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class CranDependencyPackage extends DependencyPackage ***REMOVED***
public CranDependencyPackage(Dependency d) ***REMOVED***
super(d);
***REMOVED***
***REMOVED***

65
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomDependencyPackage.java
View File

@ -1,65 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.util.Collection;
import java.util.NoSuchElementException;
import java.util.Vector;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class CustomDependencyPackage extends DependencyPackage ***REMOVED***
public CustomDependencyPackage(Dependency d) ***REMOVED***
super(d);
***REMOVED***
private String getCustomRepositoryLocation(String ansibleRoot) ***REMOVED***
return ansibleRoot+"/custom";
***REMOVED***
/*
public void serializeTo(String ansibleRoot) ***REMOVED***
for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED***
***REMOVED*** look for roles in the 'custom' repository
try ***REMOVED***
***REMOVED*** role name
String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
***REMOVED*** look for the custom role
File src = new File(this.getCustomRepositoryLocation(ansibleRoot)+"/"+roleName);
System.out.println("** CUSTOM ** " + src);
if(src.exists()) ***REMOVED***
***REMOVED*** do copy
System.out.println("copying CUSTOM role");
File dest = new File(ansibleRoot+"/work/"+roleName);
FileUtils.copyDirectory(src, dest);
***REMOVED***
***REMOVED*** catch(IOException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***
*/
public Collection<Role> getRoles(CustomRoleManager crm) ***REMOVED***
Collection<Role> out = new Vector<>();
***REMOVED*** for(String mode:new String[]***REMOVED***"add", "remove", "update"***REMOVED***) ***REMOVED***
for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update"
***REMOVED*** role name
String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
try ***REMOVED***
***REMOVED*** look for custom role
Role role = crm.getRole(roleName);
if(role!=null) ***REMOVED***
out.add(role);
***REMOVED***
***REMOVED*** catch (NoSuchElementException e) ***REMOVED***
***REMOVED*** e.printStackTrace();
System.out.println("WARNING: no custom role found for " + roleName);
***REMOVED***
***REMOVED***
return out;
***REMOVED***
***REMOVED***

32
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java
View File

@ -1,32 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.io.File;
***REMOVED***
import java.util.NoSuchElementException;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
public class CustomRoleManager ***REMOVED***
private String root;
public CustomRoleManager(String root) ***REMOVED***
this.root = root;
***REMOVED***
public String getRoot() ***REMOVED***
return this.root;
***REMOVED***
public Role getRole(String roleName) throws NoSuchElementException ***REMOVED***
File f = new File(this.getRoot(), roleName);
try ***REMOVED***
return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f);
***REMOVED*** catch (IOException e) ***REMOVED***
***REMOVED*** e.printStackTrace();
throw new NoSuchElementException("unable to find " + roleName);
***REMOVED***
***REMOVED***
***REMOVED***

55
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/DependencyPackage.java
View File

@ -1,55 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Vector;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class DependencyPackage ***REMOVED***
private Dependency dependency;
public DependencyPackage(Dependency d) ***REMOVED***
this.dependency = d;
***REMOVED***
protected Map<String, String> getDictionary(Dependency d) ***REMOVED***
Map<String, String> out = new HashMap<String, String>();
out.put("name", d.getName());
out.put("type", d.getType());
return out;
***REMOVED***
protected Dependency getDependency() ***REMOVED***
return this.dependency;
***REMOVED***
public Collection<Role> getRoles(TemplateManager tm) ***REMOVED***
Collection<Role> out = new Vector<>();
for(String mode:new String[]***REMOVED***"add"***REMOVED***) ***REMOVED*** ***REMOVED*** "remove", "update"
String roleName = this.getDependency().getType()+"-"+this.getDependency().getName().replaceAll("/", "-")+("add".equals(mode) ? "" : "-"+mode);
try ***REMOVED***
***REMOVED*** find template
Role template = tm.getRoleTemplate(this.getDependency().getType()+"-package-"+mode);
***REMOVED***
if(template!=null) ***REMOVED***
Map<String, String> dictionary = this.getDictionary(this.getDependency());
Role r = tm.fillRoleTemplate(template, dictionary);
r.setName(roleName);
out.add(r);
***REMOVED*** else ***REMOVED***
System.out.println("WARNING: template is null");
***REMOVED***
***REMOVED*** catch (NoSuchElementException e) ***REMOVED***
***REMOVED*** e.printStackTrace();
System.out.println("WARNING: no template found for " + roleName);
***REMOVED***
***REMOVED***
return out;
***REMOVED***
***REMOVED***

11
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/OSDependencyPackage.java
View File

@ -1,11 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class OSDependencyPackage extends DependencyPackage ***REMOVED***
public OSDependencyPackage(Dependency d) ***REMOVED***
super(d);
***REMOVED***
***REMOVED***

37
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java
View File

@ -1,37 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.io.File;
***REMOVED***
import java.util.Collection;
import java.util.Vector;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
public class StaticRoleManager ***REMOVED***
private String root;
public StaticRoleManager(String root) ***REMOVED***
this.root = root;
***REMOVED***
public String getRoot() ***REMOVED***
return this.root;
***REMOVED***
public Collection<Role> getStaticRoles() ***REMOVED***
Collection<Role> out = new Vector<>();
for(File f: new File(this.getRoot()).listFiles()) ***REMOVED***
try ***REMOVED***
out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f));
***REMOVED*** catch(IOException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***
return out;
***REMOVED***
***REMOVED***

95
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java
View File

@ -1,95 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.io.File;
***REMOVED***
import java.util.Map;
import java.util.NoSuchElementException;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
import org.stringtemplate.v4.ST;
public class TemplateManager ***REMOVED***
private String root;
public TemplateManager(String root) ***REMOVED***
this.root = root;
***REMOVED***
public String getTemplateRoot() ***REMOVED***
return this.root;
***REMOVED***
/**
* Read the given template
* @param templateName
* @return
* @throws IOException
*/
***REMOVED*** private String readTemplate(String templateName) throws IOException ***REMOVED***
***REMOVED*** File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml");
***REMOVED*** System.out.println("looking for file " + templateFile.getName());
***REMOVED*** String out = IOUtils.toString(new FileInputStream(templateFile), "UTF-8");
***REMOVED*** return out;
***REMOVED*** ***REMOVED***
/**
* Return the content of the given template
* @param templateName
* @return
* @throws NoSuchElementException if no such template exists
*/
***REMOVED*** public String getTemplate(String templateName) throws NoSuchElementException ***REMOVED***
***REMOVED*** String template = null;
***REMOVED*** try ***REMOVED***
***REMOVED*** template = this.readTemplate(templateName);
***REMOVED*** ***REMOVED*** catch (IOException e) ***REMOVED***
***REMOVED*** throw new NoSuchElementException();
***REMOVED*** ***REMOVED***
***REMOVED*** return template;
***REMOVED*** ***REMOVED***
public Role fillRoleTemplate(Role template, Map<String, String> dictionary) ***REMOVED***
Role out = new Role();
out.setName(template.getName());
for(RoleFile tf:template.getTaskFiles()) ***REMOVED***
out.addTaskFile(this.fillTaskTemplate(tf, dictionary));
***REMOVED***
for(RoleFile tf:template.getMeta()) ***REMOVED***
out.addMeta(this.fillTaskTemplate(tf, dictionary));
***REMOVED***
return out;
***REMOVED***
private RoleFile fillTaskTemplate(RoleFile template, Map<String, String> dictionary) ***REMOVED***
RoleFile out = new RoleFile();
out.setName(template.getName());
out.setContent(this.fillTemplate(template.getContent(), dictionary));
return out;
***REMOVED***
private String fillTemplate(String template, Map<String, String> dictionary) ***REMOVED***
if (template != null) ***REMOVED***
ST t = new ST(template);
for (String key : dictionary.keySet()) ***REMOVED***
t.add(key, dictionary.get(key));
***REMOVED***
String output = t.render();
return output;
***REMOVED***
return template;
***REMOVED***
public Role getRoleTemplate(String roleName) throws NoSuchElementException ***REMOVED***
File f = new File(this.getTemplateRoot(), roleName);
try ***REMOVED***
return AnsibleSerializeHelper.deserializeRoleFromFilesystem(f);
***REMOVED*** catch (IOException e) ***REMOVED***
***REMOVED*** e.printStackTrace();
throw new NoSuchElementException("unable to find " + roleName);
***REMOVED***
***REMOVED***
***REMOVED***

66
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -1,66 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.clients;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.Collection;
***REMOVED***
import java.util.Vector;
import org.gcube.common.resources.gcore.ServiceEndpoint;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
public class ISClient ***REMOVED***
/**
* Return the list of hosts (dataminers) in a given VRE
*
* @param vreName
* @return
*/
public Collection<Host> listDataminersInVRE() ***REMOVED***
boolean remote = false;
if (!remote) ***REMOVED***
Collection<Host> out = new Vector<>();
Host h = new Host();
***REMOVED***h.setName("bb-dataminer.res.eng.it");
***REMOVED***h.setName("vm101.ui.savba.sk");
h.setName("dataminer1-devnext.d4science.org");
out.add(h);
return out;
***REMOVED*** else ***REMOVED***
SimpleQuery query = queryFor(ServiceEndpoint.class);
***REMOVED***old version
***REMOVED***query.addCondition("$resource/Profile/Category/text() eq 'DataAnalysis'")
***REMOVED***.addCondition("$resource/Profile/Name/text() eq 'DataMiner'");
query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMiner'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> resources = client.submit(query);
Collection<Host> out = new Vector<>();
for (ServiceEndpoint r : resources) ***REMOVED***
Host h = new Host();
h.setName(r.profile().runtime().hostedOn());
out.add(h);
***REMOVED***
return out;
***REMOVED***
***REMOVED***
public static void main(String[] args) ***REMOVED***
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(a.listDataminersInVRE());
***REMOVED***
***REMOVED***

35
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Action.java
View File

@ -1,35 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
public class Action ***REMOVED***
private String name;
private String description;
private String script;
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
public String getDescription() ***REMOVED***
return description;
***REMOVED***
public void setDescription(String description) ***REMOVED***
this.description = description;
***REMOVED***
public String getScript() ***REMOVED***
return script;
***REMOVED***
public void setScript(String script) ***REMOVED***
this.script = script;
***REMOVED***
***REMOVED***

118
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java
View File

@ -1,118 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection;
import java.util.Vector;
public class Algorithm ***REMOVED***
private String name;
private String description;
private String category;
private String clazz;
private String algorithmType;
private String skipJava;
private String packageURL;
private Collection<Action> actions;
private Collection<Dependency> dependencies;
public Algorithm() ***REMOVED***
this.actions = new Vector<>();
this.dependencies = new Vector<>();
Dependency p = new Dependency();
***REMOVED***
public void addDependency(Dependency dep) ***REMOVED***
this.dependencies.add(dep);
***REMOVED***
public void addAction(Action action) ***REMOVED***
this.actions.add(action);
***REMOVED***
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
public String getDescription() ***REMOVED***
return description;
***REMOVED***
public void setDescription(String description) ***REMOVED***
this.description = description;
***REMOVED***
public String getCategory() ***REMOVED***
return category;
***REMOVED***
public void setCategory(String category) ***REMOVED***
this.category = category;
***REMOVED***
public Collection<Action> getActions() ***REMOVED***
return actions;
***REMOVED***
public Collection<Dependency> getDependencies() ***REMOVED***
return dependencies;
***REMOVED***
public void setDependencies(Collection<Dependency> deps) ***REMOVED***
this.dependencies = deps;
***REMOVED***
public String toString() ***REMOVED***
String out = "Algorithm: " + this.getName()+"\n";
out+=" Class Name: " + this.getClazz()+"\n";
out+=" Description: " + this.getDescription()+"\n";
out+=" Dependencies: " + this.getDependencies()+"\n";
return out;
***REMOVED***
public String getClazz() ***REMOVED***
return clazz;
***REMOVED***
public void setClazz(String clazz) ***REMOVED***
this.clazz = clazz;
***REMOVED***
public String getPackageURL() ***REMOVED***
return packageURL;
***REMOVED***
public void setPackageURL(String packageURL) ***REMOVED***
this.packageURL = packageURL;
***REMOVED***
public String getAlgorithmType() ***REMOVED***
return algorithmType;
***REMOVED***
public void setAlgorithmType(String algorithmType) ***REMOVED***
this.algorithmType = algorithmType;
***REMOVED***
public String getSkipJava() ***REMOVED***
return skipJava;
***REMOVED***
public void setSkipJava(String skipJava) ***REMOVED***
this.skipJava = skipJava;
***REMOVED***
***REMOVED***

49
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/AlgorithmSet.java
View File

@ -1,49 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection;
import java.util.Vector;
public class AlgorithmSet ***REMOVED***
private String name;
private Collection<Algorithm> algorithms;
public AlgorithmSet() ***REMOVED***
this.algorithms = new Vector<>();
***REMOVED***
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
public Collection<Algorithm> getAlgorithms() ***REMOVED***
return new Vector<>(algorithms);
***REMOVED***
public void addAlgorithm(Algorithm algoritm) ***REMOVED***
this.algorithms.add(algoritm);
***REMOVED***
public Boolean hasAlgorithm(Algorithm algorithm) ***REMOVED***
for (Algorithm a : this.algorithms) ***REMOVED***
if (a.getName().equals(algorithm.getName())) ***REMOVED***
return true;
***REMOVED***
***REMOVED***
return false;
***REMOVED***
public String toString() ***REMOVED***
String out = "ALGOSET: " + this.name + "\n";
for(Algorithm a:this.algorithms) ***REMOVED***
out+=a+"\n";
***REMOVED***
return out;
***REMOVED***
***REMOVED***

73
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Cluster.java
View File

@ -1,73 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection;
import java.util.Vector;
public class Cluster ***REMOVED***
/**
* The set of hosts belonging to the cluster.
*/
private Collection<Host> hosts;
/**
* A name for this cluster.
*/
private String name;
/**
* A description of this cluster.
*/
private String description;
/**
* The set of algorithms deployed on this cluster (i.e. on all its hosts)
*/
private Collection<AlgorithmSet> algoSets;
public Cluster() ***REMOVED***
this.hosts = new Vector<>();
this.algoSets = new Vector<>();
***REMOVED***
public void addAlgorithmSet(AlgorithmSet set) ***REMOVED***
this.algoSets.add(set);
***REMOVED***
public void addHost(Host host) ***REMOVED***
this.hosts.add(host);
***REMOVED***
public Collection<Host> getHosts() ***REMOVED***
return hosts;
***REMOVED***
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
public String getDescription() ***REMOVED***
return description;
***REMOVED***
public void setDescription(String description) ***REMOVED***
this.description = description;
***REMOVED***
public Collection<AlgorithmSet> getAlgorithmSets() ***REMOVED***
return algoSets;
***REMOVED***
public String toString() ***REMOVED***
String out = "Cluster: "+this.name+"\n";
for(Host h:this.getHosts()) ***REMOVED***
out+=" "+h+"\n";
***REMOVED***
return out;
***REMOVED***
***REMOVED***

29
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Dependency.java
View File

@ -1,29 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
public class Dependency ***REMOVED***
private String name;
private String type;
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
public String getType() ***REMOVED***
return type;
***REMOVED***
public void setType(String type) ***REMOVED***
this.type = type;
***REMOVED***
public String toString() ***REMOVED***
return this.type+":"+this.name;
***REMOVED***
***REMOVED***

15
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Domain.java
View File

@ -1,15 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
public class Domain ***REMOVED***
private String name;
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
***REMOVED***

39
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Host.java
View File

@ -1,39 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
public class Host ***REMOVED***
private String name;
private Domain domain;
public Host() ***REMOVED***
***REMOVED***
public String getFullyQualifiedName() ***REMOVED***
if(this.domain!=null && this.domain.getName()!=null)
return this.getName()+"."+this.getDomain().getName();
else
return this.getName();
***REMOVED***
public String getName() ***REMOVED***
return name;
***REMOVED***
public void setName(String name) ***REMOVED***
this.name = name;
***REMOVED***
public Domain getDomain() ***REMOVED***
return domain;
***REMOVED***
public void setDomain(Domain domain) ***REMOVED***
this.domain = domain;
***REMOVED***
public String toString() ***REMOVED***
return this.name + "@" + this.domain;
***REMOVED***
***REMOVED***

15
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/AlgorithmComparator.java
View File

@ -1,15 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator;
import java.util.Comparator;
***REMOVED***
public class AlgorithmComparator implements Comparator<Algorithm> ***REMOVED***
@Override
public int compare(Algorithm a1, Algorithm a2) ***REMOVED***
return a1.getName().compareTo(a2.getName());
***REMOVED***
***REMOVED***

18
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/DependencyComparator.java
View File

@ -1,18 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator;
import java.util.Comparator;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class DependencyComparator implements Comparator<Dependency> ***REMOVED***
@Override
public int compare(Dependency a1, Dependency a2) ***REMOVED***
int out = a1.getType().compareTo(a2.getType());
if(out!=0)
return out;
return a1.getName().compareTo(a2.getName());
***REMOVED***
***REMOVED***

17
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/comparator/HostComparator.java
View File

@ -1,17 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator;
import java.util.Comparator;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
public class HostComparator implements Comparator<Host> ***REMOVED***
@Override
public int compare(Host h1, Host h2) ***REMOVED***
int out = h1.getDomain().getName().compareTo(h2.getDomain().getName());
if(out!=0)
return out;
return h1.getName().compareTo(h2.getName());
***REMOVED***
***REMOVED***

103
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AddAlgorithmCommand.java
View File

@ -1,103 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.process;
import java.util.StringTokenizer;
public class AddAlgorithmCommand ***REMOVED***
private String command;
private String name;
private String category;
private String clazz;
private String scope;
private String algorithmType;
private String skipJava;
private String url;
private String description;
public AddAlgorithmCommand(String cmd) ***REMOVED***
StringTokenizer st = new StringTokenizer(cmd, " ");
if (st.hasMoreElements())
command = st.nextToken();
if (st.hasMoreElements())
name = st.nextToken();
if (st.hasMoreElements())
category = st.nextToken();
if (st.hasMoreElements())
clazz = st.nextToken();
if (st.hasMoreElements())
scope = st.nextToken();
if (st.hasMoreElements())
algorithmType = st.nextToken();
if (st.hasMoreElements())
skipJava = st.nextToken();
if (st.hasMoreElements())
url = st.nextToken();
String d = "";
while (st.hasMoreElements())
d = d + st.nextToken() + " ";
this.setDescription(d);
***REMOVED***
public void setDescription(String d) ***REMOVED***
if(d!=null) ***REMOVED***
d = d.trim();
if(d.startsWith("\"") && d.endsWith("\"")) ***REMOVED***
d = d.substring(1, d.length()-1).trim();
***REMOVED***
***REMOVED***
this.description = d;
***REMOVED***
public String getCommand() ***REMOVED***
return command;
***REMOVED***
public String getName() ***REMOVED***
return name;
***REMOVED***
public String getCategory() ***REMOVED***
return category;
***REMOVED***
public String getClazz() ***REMOVED***
return clazz;
***REMOVED***
public String getVRE() ***REMOVED***
return scope;
***REMOVED***
public String getAlgorithmType() ***REMOVED***
return algorithmType;
***REMOVED***
public String getSkipjava() ***REMOVED***
return skipJava;
***REMOVED***
public String getUrl() ***REMOVED***
return url;
***REMOVED***
public String getDescription() ***REMOVED***
return description;
***REMOVED***
public String toString() ***REMOVED***
String out = "";
out += String.format("%-12s: %s\n", "command", command);
out += String.format("%-12s: %s\n", "algo name", name);
out += String.format("%-12s: %s\n", "category", category);
out += String.format("%-12s: %s\n", "class", clazz);
out += String.format("%-12s: %s\n", "scope", scope);
out += String.format("%-12s: %s\n", "algo type", algorithmType);
out += String.format("%-12s: %s\n", "skip java", skipJava);
out += String.format("%-12s: %s\n", "url", url);
out += String.format("%-12s: %s\n", "description", this.description);
return out;
***REMOVED***
***REMOVED***

290
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java
View File

@ -1,290 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.process;
***REMOVED***
import java.io.InputStream;
***REMOVED***
import java.util.HashMap;
***REMOVED***
import java.util.Map;
import java.util.Vector;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.lang.StringEscapeUtils;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
public class AlgorithmPackageParser ***REMOVED***
/**
* The name of the file containing algorithm metadata. Expected in the root
* directory of the package.
*/
private static final String METADATA_FILE_NAME = "Info.txt";
private static final String METADATA_ALGORITHM_NAME = "Algorithm Name";
private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description";
private static final String METADATA_CLASS_NAME = "Class Name";
private static final String METADATA_PACKAGES = "Packages";
private static final String METADATA_KEY_VALUE_SEPARATOR = ":";
private static final int BUFFER_SIZE = 4096;
/**
* Given an URL to an algorithm package, create an Algorithm object with its
* metadata. Metadata are extracted from the 'info.txt' file, if any, in the
* package.
*
* @param url
* @return An Algorithm object or null if no 'info.txt' is found in the
* package.
* @throws IOException
*/
public Algorithm parsePackage(String url) throws IOException ***REMOVED***
String packageMetadata = this.getPackageMetadata(url);
if (packageMetadata == null) ***REMOVED***
System.out.println("WARNING: No metadata found for " + url);
return null;
***REMOVED*** else ***REMOVED***
Map<String, List<String>> parsedMetadata = this.parseMetadata(packageMetadata);
Algorithm a = this.createAlgorithm(parsedMetadata);
a.setPackageURL(url);
return a;
***REMOVED***
***REMOVED***
/**
* Extract the content of the metadata file from the package.
*
* @param url
* @return
* @throws IOException
*/
private String getPackageMetadata(String url) throws IOException ***REMOVED***
InputStream is = new URL(url).openStream();
ZipInputStream zipIs = new ZipInputStream(is);
ZipEntry entry = zipIs.getNextEntry();
String out = null;
while (entry != null) ***REMOVED***
if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) ***REMOVED***
out = this.getEntryContent(zipIs);
break;
***REMOVED***
entry = zipIs.getNextEntry();
***REMOVED***
is.close();
zipIs.close();
return out;
***REMOVED***
/**
* Read the content of a zip entry and place it in a string.
* @param zipIn
* @return
* @throws IOException
*/
private String getEntryContent(ZipInputStream zipIn) throws IOException ***REMOVED***
StringBuilder s = new StringBuilder();
byte[] buffer = new byte[BUFFER_SIZE];
int read = 0;
while ((read = zipIn.read(buffer)) != -1) ***REMOVED***
s.append(new String(buffer, 0, read));
***REMOVED***
return s.toString();
***REMOVED***
/**
* Parse the content of the metadata file and create a key+multivalue map.
* @param metadata
* @return
*/
private Map<String, List<String>> parseMetadata(String metadata) ***REMOVED***
Map<String, List<String>> out = new HashMap<String, List<String>>();
String[] lines = metadata.split("\n");
String key = null;
String value = null;
for (String line : lines) ***REMOVED***
***REMOVED*** skip empty lines
if (line.trim().isEmpty()) ***REMOVED***
continue;
***REMOVED***
***REMOVED*** scan lines one by one, looking for key and values
String[] parts = line.split(METADATA_KEY_VALUE_SEPARATOR);
if (parts.length > 1) ***REMOVED***
***REMOVED*** key and value on the same line
key = parts[0].trim();
value = line.substring(parts[0].length() + 1).trim();
***REMOVED*** else if (parts.length == 1) ***REMOVED***
***REMOVED*** either a key or a value
if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) ***REMOVED***
***REMOVED*** key
key = parts[0].trim();
value = null;
***REMOVED*** else ***REMOVED***
***REMOVED*** value
value = line.trim();
***REMOVED***
***REMOVED***
***REMOVED*** add key+value to the map
if (key != null && value != null) ***REMOVED***
List<String> values = out.get(key);
if (values == null) ***REMOVED***
values = new Vector<>();
out.put(key, values);
***REMOVED***
values.add(value);
System.out.println(key + METADATA_KEY_VALUE_SEPARATOR + " " + values);
***REMOVED***
***REMOVED***
return out;
***REMOVED***
/**
* Create an Algorithm starting from its metadata
* @param metadata
* @return
*/
***REMOVED*** private Algorithm createAlgorithm(Map<String, List<String>> metadata) ***REMOVED***
***REMOVED*** Algorithm out = new Algorithm();
***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
***REMOVED*** out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
***REMOVED*** out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
***REMOVED*** List<String> dependencies = extractMultipleValues(metadata, METADATA_PACKAGES);
***REMOVED*** if (dependencies != null) ***REMOVED***
***REMOVED*** for (String pkg : dependencies) ***REMOVED***
***REMOVED*** Dependency dep = new Dependency();
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("os");
***REMOVED*** out.addDependency(dep);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return out;
***REMOVED*** ***REMOVED***
private Algorithm createAlgorithm(Map<String, List<String>> metadata) ***REMOVED***
Algorithm out = new Algorithm();
out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
***REMOVED***List<String> dependencies = extractMultipleValues(metadata, METADATA_PACKAGES);
List<String> rdependencies = extractMultipleValues(metadata, "cran");
if (rdependencies != null) ***REMOVED***
for (String pkg : rdependencies) ***REMOVED***
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:"))***REMOVED***
dep.setName(pkg);
dep.setType("cran");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> defdependencies = extractMultipleValues(metadata, "Packages");
if (defdependencies != null) ***REMOVED***
for (String pkg : defdependencies) ***REMOVED***
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:"))***REMOVED***
dep.setName(pkg);
dep.setType("os");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> osdependencies = extractMultipleValues(metadata, "os");
if (osdependencies != null) ***REMOVED***
for (String pkg : osdependencies) ***REMOVED***
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:"))***REMOVED***
dep.setName(pkg);
dep.setType("os");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> gitdependencies = extractMultipleValues(metadata, "github");
if (gitdependencies != null) ***REMOVED***
for (String pkg : gitdependencies) ***REMOVED***
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:"))***REMOVED***
dep.setName(pkg);
dep.setType("github");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> cdependencies = extractMultipleValues(metadata, "custom");
if (cdependencies != null) ***REMOVED***
for (String pkg : cdependencies) ***REMOVED***
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:"))***REMOVED***
dep.setName(pkg);
dep.setType("custom");
out.addDependency(dep);
***REMOVED***
***REMOVED***
***REMOVED*** if (pkg.startsWith("r:"))***REMOVED***
***REMOVED*** ***REMOVED***String results = StringEscapeUtils.escapeJava(pkg);
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("cran");
***REMOVED*** ***REMOVED***
***REMOVED*** if (pkg.startsWith("custom:"))***REMOVED***
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("custom");
***REMOVED*** ***REMOVED***
***REMOVED*** if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:"))***REMOVED***
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("os");
***REMOVED*** ***REMOVED***
return out;
***REMOVED***
private static String extractSingleValue(Map<String, List<String>> metadata,
String key) ***REMOVED***
List<String> l = metadata.get(key);
if (l != null && l.size() == 1) ***REMOVED***
return l.get(0);
***REMOVED*** else ***REMOVED***
return null;
***REMOVED***
***REMOVED***
private static List<String> extractMultipleValues(
Map<String, List<String>> metadata, String key) ***REMOVED***
List<String> l = metadata.get(key);
if (l != null) ***REMOVED***
return new Vector<>(l);
***REMOVED*** else ***REMOVED***
return null;
***REMOVED***
***REMOVED***
***REMOVED***

24
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
View File

@ -1,24 +0,0 @@
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
public interface PoolManager ***REMOVED***
String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException;
Algorithm extractAlgorithm(String url) throws IOException;
String getLogById(String logId) throws IOException;
void getLogId(Algorithm algo, String vre);
String getScriptFromURL(URL logId) throws IOException;
URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException;
***REMOVED***

262
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -1,262 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service;
import java.io.BufferedReader;
import java.io.File;
***REMOVED***
import java.io.InputStreamReader;
***REMOVED***
***REMOVED***
import java.net.URLConnection;
***REMOVED***
import java.util.UUID;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
import org.gcube.dataanalysis.dataminer.poolmanager.rest.PoolManager;
***REMOVED***
***REMOVED***
public class DataminerPoolManager implements PoolManager ***REMOVED***
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
***REMOVED*** static Collection<Algorithm> algorithms;
***REMOVED***
***REMOVED*** static Collection<AlgorithmSet> sets;
***REMOVED***
***REMOVED*** static ***REMOVED***
***REMOVED*** algorithms = new Vector<>();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public DataminerPoolManager() ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add a new algorithm to the set of known ones. No further action is expected
***REMOVED*** * on the pool.
***REMOVED*** */
***REMOVED*** public void publishAlgorithm(Algorithm algorithm) ***REMOVED***
***REMOVED*** algorithms.add(algorithm);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Re-deploy the given algorithm wherever it's installed
***REMOVED*** *
***REMOVED*** * @param algorithm
***REMOVED*** */
***REMOVED*** /*
***REMOVED*** * public void updateAlgorithm(Algorithm algorithm) ***REMOVED*** ***REMOVED*** TODO implement this ***REMOVED***
***REMOVED*** */
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Add the give algorithm to the given set
***REMOVED*** *
***REMOVED*** * @param algorithmId
***REMOVED*** * @param setId
***REMOVED*** */
***REMOVED*** public void addAlgorithmToSet(String algorithmName, String setName) ***REMOVED***
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Algorithm algorithm = this.getAlgorithm(algorithmName);
***REMOVED*** if (set != null && algorithm != null) ***REMOVED***
***REMOVED*** set.addAlgorithm(algorithm);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** /**
***REMOVED*** * Apply the given set of algorithms to the given cluster
***REMOVED*** *
***REMOVED*** * @param setId
***REMOVED*** * @param clusterId
***REMOVED*** */
***REMOVED*** public void applyAlgorithmSetToCluster(String setName, String clusterName) ***REMOVED***
***REMOVED*** AlgorithmSet set = this.getAlgorithmSet(setName);
***REMOVED*** Cluster cluster = new ISClient().getCluster(clusterName);
***REMOVED*** if (set != null && cluster != null) ***REMOVED***
***REMOVED*** cluster.addAlgorithmSet(set);
***REMOVED*** this.updateClusters();
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private AlgorithmSet getAlgorithmSet(String name) ***REMOVED***
***REMOVED*** for (AlgorithmSet set : sets) ***REMOVED***
***REMOVED*** if (name.equals(set.getName())) ***REMOVED***
***REMOVED*** return set;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** private Algorithm getAlgorithm(String name) ***REMOVED***
***REMOVED*** for (Algorithm a : algorithms) ***REMOVED***
***REMOVED*** if (name.equals(a.getName())) ***REMOVED***
***REMOVED*** return a;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public void getLogId(final Algorithm algorithm, final String vre) ***REMOVED***
***REMOVED*** new Thread() ***REMOVED***
***REMOVED*** public void run() ***REMOVED***
***REMOVED*** while (true) ***REMOVED***
***REMOVED*** try ***REMOVED***
***REMOVED*** addAlgorithmToVRE(algorithm, vre);
***REMOVED*** ***REMOVED*** catch (Exception e) ***REMOVED***
***REMOVED*** ***REMOVED***log here
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***.start();
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public String getLogId()***REMOVED***
***REMOVED*** PrintStream console = System.out;
***REMOVED*** File path = new File(worker.getWorkdir() + File.separator + "logs");
***REMOVED*** path.mkdirs();
***REMOVED*** File n = new File(path + File.separator + worker.getWorkerId());
***REMOVED*** FileOutputStream fos = new FileOutputStream(n);
***REMOVED*** PrintStream ps = new PrintStream(fos);
***REMOVED*** System.setOut(ps);
***REMOVED*** worker.apply();
***REMOVED*** System.setOut(console);
***REMOVED*** worker.apply();
***REMOVED*** System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED******REMOVED***
***REMOVED*** public String getLogById(String id) throws IOException ***REMOVED***
***REMOVED*** String strLine = null;
***REMOVED*** try***REMOVED***
***REMOVED*** FileInputStream fstream = new FileInputStream("/tmp/dataminer-pool-manager/work/"+id+"/logs/"+id);
***REMOVED*** BufferedReader br = new BufferedReader(new InputStreamReader(fstream));
***REMOVED*** /* read log line by line */
***REMOVED*** while ((strLine = br.readLine()) != null) ***REMOVED***
***REMOVED*** /* parse strLine to obtain what you want */
***REMOVED*** System.out.println (strLine);
***REMOVED*** ***REMOVED***
***REMOVED*** br.close();
***REMOVED*** ***REMOVED*** catch (Exception e) ***REMOVED***
***REMOVED*** System.err.println("Error: " + e.getMessage());
***REMOVED*** ***REMOVED***
***REMOVED*** return strLine;
***REMOVED******REMOVED***
public String getScriptFromURL(URL url) throws IOException ***REMOVED***
if (url == null) ***REMOVED***
return null;
***REMOVED***
URLConnection yc = url.openConnection();
BufferedReader input = new BufferedReader(new InputStreamReader(
yc.getInputStream()));
String line;
StringBuffer buffer = new StringBuffer();
while ((line = input.readLine()) != null) ***REMOVED***
buffer.append(line + "\n");
***REMOVED***
String bufferScript = buffer.substring(0, buffer.length());
input.close();
return bufferScript;
***REMOVED***
/**
* Publish the given algorithm in the given VRE
*
* @param algorithmName
* @param vre
*
*/
public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException ***REMOVED***
***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet();
algoSet.setName("fake");
algoSet.addAlgorithm(algorithm);
final String uuid = UUID.randomUUID().toString();
new Thread(new Runnable() ***REMOVED***
***REMOVED***
public void run() ***REMOVED***
***REMOVED***
try ***REMOVED***
addAlgorithmsToVRE(algoSet, vre, uuid);
***REMOVED*** catch (IOException e) ***REMOVED***
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***).start();
***REMOVED***this line will execute immediately, not waiting for your task to complete
System.out.println(uuid);
return uuid;
***REMOVED***
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException***REMOVED***
File path = new File(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager/work/"+a+File.separator+"logs");
path.mkdirs();
File n = new File(path + File.separator +a);
***REMOVED***String addr = InetAddress.getLocalHost().getHostAddress();
return new File(n.getPath()).toURI().toURL();
***REMOVED***
public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException ***REMOVED***
***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
for(Host h:new ISClient().listDataminersInVRE()) ***REMOVED***
cluster.addHost(h);
***REMOVED***
***REMOVED*** apply the changes
AnsibleBridge a = new AnsibleBridge();
return a.applyAlgorithmSetToCluster(algorithms, cluster,uuid).getWorkerId();
***REMOVED***
public Algorithm extractAlgorithm(String url) throws IOException ***REMOVED***
return new AlgorithmPackageParser().parsePackage(url);
***REMOVED***
@Override
public void getLogId(Algorithm algo, String vre) ***REMOVED***
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
@Override
public String getLogById(String logId) throws IOException ***REMOVED***
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
***REMOVED***

145
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/PropertiesBasedProxySelector.java
View File

@ -1,145 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
***REMOVED***
import java.net.Authenticator;
import java.net.InetSocketAddress;
import java.net.PasswordAuthentication;
import java.net.Proxy;
import java.net.ProxySelector;
import java.net.SocketAddress;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
***REMOVED***
import org.apache.commons.configuration.ConfigurationException;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.configuration.reloading.FileChangedReloadingStrategy;
interface NetworkConfiguration ***REMOVED***
public String getProxyHost();
public String getProxyPort();
public String getProxyUser();
public String getProxyPassword();
public String getNonProxyHosts();
***REMOVED***
class FileBasedProxyConfiguration implements NetworkConfiguration ***REMOVED***
private static PropertiesConfiguration configuration;
public FileBasedProxyConfiguration(String path) ***REMOVED***
try ***REMOVED***
***REMOVED*** load the configuration
configuration = new PropertiesConfiguration(path);
***REMOVED*** set the reloading strategy to enable hot-configuration
FileChangedReloadingStrategy fcrs = new FileChangedReloadingStrategy();
configuration.setReloadingStrategy(fcrs);
***REMOVED*** catch (ConfigurationException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***
@Override
public String getProxyHost() ***REMOVED***
return configuration.getString("proxyHost");
***REMOVED***
@Override
public String getProxyPort() ***REMOVED***
return configuration.getString("proxyPort");
***REMOVED***
@Override
public String getProxyUser() ***REMOVED***
return configuration.getString("proxyUser");
***REMOVED***
@Override
public String getProxyPassword() ***REMOVED***
return configuration.getString("proxyPassword");
***REMOVED***
@Override
public String getNonProxyHosts() ***REMOVED***
return configuration.getString("nonProxyHosts");
***REMOVED***
***REMOVED***
public class PropertiesBasedProxySelector extends ProxySelector ***REMOVED***
List<Proxy> proxies = null;
List<String> nonProxyHosts = null;
public PropertiesBasedProxySelector(String proxySettingsPath) ***REMOVED***
this(new FileBasedProxyConfiguration(proxySettingsPath));
***REMOVED***
public PropertiesBasedProxySelector(NetworkConfiguration config) ***REMOVED***
if (config == null || config.getProxyHost() == null) ***REMOVED***
this.proxies = null;
return;
***REMOVED***
String host = config.getProxyHost();
int port = 80;
if (config.getProxyPort() != null) ***REMOVED***
port = Integer.valueOf(config.getProxyPort());
***REMOVED***
if (config.getNonProxyHosts() != null) ***REMOVED***
this.nonProxyHosts = Arrays
.asList(config.getNonProxyHosts().split("\\|"));
***REMOVED***
this.proxies = new ArrayList<Proxy>();
this.proxies.add(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(host,
port)));
if (config.getProxyUser() != null) ***REMOVED***
final String username = config.getProxyUser();
final String password = config.getProxyPassword();
Authenticator.setDefault(new Authenticator() ***REMOVED***
@Override
protected PasswordAuthentication getPasswordAuthentication() ***REMOVED***
return new PasswordAuthentication(username, password.toCharArray());
***REMOVED***
***REMOVED***);
***REMOVED***
***REMOVED***
@Override
public List<Proxy> select(URI uri) ***REMOVED***
if (this.nonProxyHosts == null) ***REMOVED***
return Arrays.asList(Proxy.NO_PROXY);
***REMOVED*** else ***REMOVED***
for (String entry : this.nonProxyHosts) ***REMOVED***
entry = entry.trim();
if (entry.startsWith("*") && uri.getHost().endsWith(entry.substring(1))) ***REMOVED***
return Arrays.asList(Proxy.NO_PROXY);
***REMOVED***
if (uri.getHost().equals(entry)) ***REMOVED***
return Arrays.asList(Proxy.NO_PROXY);
***REMOVED***
***REMOVED***
return this.proxies;
***REMOVED***
***REMOVED***
@Override
public void connectFailed(URI uri, SocketAddress socketAddress, IOException e) ***REMOVED***
***REMOVED***
***REMOVED***

13
src/main/webapp/WEB-INF/gcube-app.xml
View File

@ -1,13 +0,0 @@
<application mode="online">
<name>dataminer-pool-manager</name>
<group>dataanalysis</group>
<version>0.0.1</version>
<!--<description>Lorem ipsum dolor sit amet...</description>-->
<!--<persistence location="/some/custom/location</persistence" />-->
<!-- <exclude>*</exclude> -->
</application>

11
src/main/webapp/WEB-INF/gcube-handlers.xml
View File

@ -1,11 +0,0 @@
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<handlers>
<lifecycle>
<profile-management />
<!-- <plugin-registration-handler /> -->
</lifecycle>
<request>
<request-validation />
<request-accounting />
</request>
</handlers>

24
src/main/webapp/WEB-INF/web.xml
View File

@ -1,24 +0,0 @@
<!DOCTYPE web-app PUBLIC
"-***REMOVED***Sun Microsystems, Inc.***REMOVED***DTD Web Application 2.3***REMOVED***EN"
"http:***REMOVED***java.sun.com/dtd/web-app_2_3.dtd" >
<web-app>
<display-name>Archetype Created Web Application</display-name>
<servlet>
<servlet-name>REST-API</servlet-name>
<servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>jersey.config.server.provider.packages</param-name>
<param-value>org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>REST-API</servlet-name>
<url-pattern>/rest/*</url-pattern>
</servlet-mapping>
</web-app>

117
src/test/java/AlgorithmPackageParserTest.java
View File

@ -1,117 +0,0 @@
***REMOVED***
import java.io.InputStream;
***REMOVED***
***REMOVED***
import java.util.Vector;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand;
import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
public class AlgorithmPackageParserTest ***REMOVED***
private static int BUFFER_SIZE = 2048;
public void extractAllAlgorithms() throws IOException ***REMOVED***
String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms";
List<String> commands = this.extractAddAlgorithmCommands(url);
AlgorithmSet algorithms = new AlgorithmSet();
for (String cmd : commands) ***REMOVED***
System.out.println("-----------------------------------------");
System.out.println(cmd);
AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd);
System.out.println(aac);
***REMOVED*** start creating the algo from the command
Algorithm algo = new Algorithm();
algo.setAlgorithmType(aac.getAlgorithmType());
algo.setCategory(aac.getCategory());
algo.setClazz(aac.getClazz());
algo.setDescription(aac.getDescription());
algo.setName(aac.getName());
algo.setPackageURL(aac.getUrl());
algo.setSkipJava(aac.getSkipjava());
***REMOVED*** then override with info from the package
if (aac.getUrl().length() > 4) ***REMOVED***
Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl());
if (packagedAlgo != null) ***REMOVED***
algo.setDependencies(packagedAlgo.getDependencies());
***REMOVED***
***REMOVED***
algorithms.addAlgorithm(algo);
break;
***REMOVED***
***REMOVED***to uncomment
***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms,
***REMOVED*** "/gcube/devNext/NextNext");
***REMOVED***
/**
* Extract 'addAlgorithm' commands from a file containing wiki-table-style
* entries for algorithm.
*
* @return
* @throws IOException
*/
private List<String> extractAddAlgorithmCommands(String listUrl)
throws IOException ***REMOVED***
URL url = new URL(listUrl);
InputStream is = url.openStream();
StringBuilder s = new StringBuilder();
byte[] buffer = new byte[BUFFER_SIZE];
int read = 0;
while ((read = is.read(buffer)) != -1) ***REMOVED***
s.append(new String(buffer, 0, read));
***REMOVED***
List<String> out = new Vector<>();
String[] lines = s.toString().split("\n");
for (String line : lines) ***REMOVED***
System.out.println("--------------------");
if (!line.isEmpty()) ***REMOVED***
String[] parts = line.split("\\|");
int c = 1;
for (String part : parts) ***REMOVED***
if (part == null || part.trim().isEmpty()) ***REMOVED***
continue;
***REMOVED***
System.out.println(c + ". " + part);
c++;
if (part.contains("addAlgorithm.sh")) ***REMOVED***
String cmd = part.trim();
cmd = cmd.replaceAll("<notextile>", "");
cmd = cmd.replaceAll("</notextile>", "");
System.out.println(cmd);
***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd);
***REMOVED*** System.out.println(aac);
out.add(cmd);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
return out;
***REMOVED***
/**
* Create an Algorithm starting from the algorithm jar.
*
* @param url
* @return
* @throws IOException
*/
private Algorithm extractAlgorithm(String url) throws IOException ***REMOVED***
return new AlgorithmPackageParser().parsePackage(url);
***REMOVED***
public static void main(String[] args) throws Exception ***REMOVED***
***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector(
***REMOVED*** "/home/ngalante/.proxy-settings"));
new AlgorithmPackageParserTest().extractAllAlgorithms();
***REMOVED***
***REMOVED***

33
src/test/java/AnsibleWorkerTest.java
View File

@ -1,33 +0,0 @@
import java.io.File;
***REMOVED***
import java.util.UUID;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
public class AnsibleWorkerTest ***REMOVED***
public static void main(String[] args) throws IOException ***REMOVED***
AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString()));
System.out.println("created worker named " + worker.getWorkerId());
worker.setInventory(new Inventory());
worker.setPlaybook(new Playbook());
Role r = new Role();
r.setName("latex");
RoleFile tf = new RoleFile("main", "do something special for " + r.getName());
r.addTaskFile(tf);
worker.addRole(r);
worker.apply();
***REMOVED***
***REMOVED***

227
src/test/java/DataminerPoolManagerTest.java
View File

@ -1,227 +0,0 @@
import java.net.ProxySelector;
import java.util.UUID;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class DataminerPoolManagerTest ***REMOVED***
/*
private static Map<String, Domain> domains = new HashMap<>();
private static Map<String, Cluster> clusters = new HashMap<>();
private static Dependency createDependency(String depName) ***REMOVED***
String[] parts = depName.split(":");
Dependency out = new Dependency();
if(parts.length>1) ***REMOVED***
out.setType(parts[0]);
out.setName(parts[1]);
***REMOVED*** else ***REMOVED***
out.setType("os");
out.setName(depName);
***REMOVED***
return out;
***REMOVED***
private static Algorithm createAlgorithm(String name, String ... deps) ***REMOVED***
Algorithm a = new Algorithm();
a.setName(name);
for(String dep:deps) ***REMOVED***
a.addDependency(createDependency(dep));
***REMOVED***
return a;
***REMOVED***
private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) ***REMOVED***
AlgorithmSet out = new AlgorithmSet();
out.setName(name);
for(Algorithm a:algs) ***REMOVED***
out.addAlgorithm(a);
***REMOVED***
return out;
***REMOVED***
private static Domain getDomain(String name) ***REMOVED***
if(domains.get(name)==null) ***REMOVED***
Domain d = new Domain();
d.setName(name);
domains.put(name, d);
return d;
***REMOVED*** else ***REMOVED***
return domains.get(name);
***REMOVED***
***REMOVED***
private static Host createHost(String hostname, String domainName) ***REMOVED***
Host out = new Host();
out.setName(hostname);
Domain d = getDomain(domainName);
out.setDomain(d);
return out;
***REMOVED***
private static Cluster getCluster(String name) ***REMOVED***
if(clusters.get(name)==null) ***REMOVED***
Cluster d = new Cluster();
d.setName(name);
clusters.put(name, d);
return d;
***REMOVED*** else ***REMOVED***
return clusters.get(name);
***REMOVED***
***REMOVED***
private static Collection<Dependency> extractDependencies() ***REMOVED***
Collection<Dependency> out = new TreeSet<>(new DependencyComparator());
for(Cluster c:clusters.values()) ***REMOVED***
for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED***
for(Algorithm a:as.getAlgorithms()) ***REMOVED***
for(Dependency d:a.getDependencies()) ***REMOVED***
out.add(d);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
return out;
***REMOVED***
private static Collection<Algorithm> extractAlgorithms() ***REMOVED***
Collection<Algorithm> out = new TreeSet<>(new AlgorithmComparator());
for(Cluster c:clusters.values()) ***REMOVED***
for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED***
for(Algorithm a:as.getAlgorithms()) ***REMOVED***
out.add(a);
***REMOVED***
***REMOVED***
***REMOVED***
return out;
***REMOVED***
static ***REMOVED***
Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package");
Algorithm ensemble = createAlgorithm("ensemble", "python");
Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah");
AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe);
AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble);
AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe);
Cluster cluster1 = getCluster("cluster-1");
cluster1.addHost(createHost("host1", "domain1"));
cluster1.addHost(createHost("host2", "domain1"));
cluster1.addHost(createHost("host3", "domain1"));
cluster1.addHost(createHost("host1", "domain2"));
cluster1.addHost(createHost("host2", "domain2"));
Cluster cluster2 = getCluster("cluster-2");
cluster2.addHost(createHost("host4", "domain1"));
cluster2.addHost(createHost("host5", "domain1"));
cluster2.addHost(createHost("host6", "domain1"));
cluster2.addHost(createHost("host3", "domain2"));
cluster2.addHost(createHost("host4", "domain2"));
cluster2.addHost(createHost("host5", "domain2"));
cluster1.addAlgorithmSet(as1);
cluster1.addAlgorithmSet(as2);
cluster2.addAlgorithmSet(as1);
cluster2.addAlgorithmSet(as3);
***REMOVED***
*/
public static void main(String[] args) throws Exception ***REMOVED***
/*
AnsibleBridge ab = new AnsibleBridge();
ab.printInventoryByDomainAndSets(clusters.values());
System.out.println("-----------");
ab.printInventoryBySets(clusters.values());
AnsibleWorker worker = ab.createWorker();
for(Algorithm a:extractAlgorithms()) ***REMOVED***
for(Role r:ab.generateRoles(a)) ***REMOVED***
worker.addRole(r);
***REMOVED***
***REMOVED***
for(Dependency d:extractDependencies()) ***REMOVED***
for(Role r:ab.generateRoles(d)) ***REMOVED***
worker.addRole(r);
***REMOVED***
***REMOVED***
*/
ScopeProvider.instance.set("/gcube/devNext/NextNext");
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
***REMOVED***
***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc)
***REMOVED*** Algorithm algorithm = new Algorithm();
***REMOVED*** algorithm.setName("ichtyop");
***REMOVED*** algorithm.setClazz("org.gcube...");
***REMOVED*** algorithm.setDescription("some description");
***REMOVED***
***REMOVED*** Dependency d = new Dependency();
***REMOVED*** d.setName("libpng");
***REMOVED*** d.setType("os");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("some-r-package");
***REMOVED*** d.setType("cran");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("some-other-r-package");
***REMOVED*** d.setType("cran");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("voodoo");
***REMOVED*** d.setType("custom");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc)
***REMOVED*** Algorithm ewe = new Algorithm();
***REMOVED*** ewe.setName("ewe");
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("voodoo");
***REMOVED*** d.setType("custom");
***REMOVED*** ewe.addDependency(d);
AlgorithmSet algorithms = new AlgorithmSet();
algorithms.setName("dummy-set");
***REMOVED*** algorithms.addAlgorithm(algorithm);
***REMOVED*** algorithms.addAlgorithm(ewe);
Algorithm ensemble = new Algorithm();
ensemble.setName("ewe");
ensemble.setCategory("a");
ensemble.setAlgorithmType("transducerers");
ensemble.setPackageURL("http:***REMOVED***data.d4science.org/eDd5b2ovMmpSVEZaZWYvY3g2bDZNcGRFNUIxNi85UFlHbWJQNStIS0N6Yz0");
ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone");
ensemble.setSkipJava("N");
ensemble.setDescription("test");
Dependency d = new Dependency();
d.setName("libpng3");
d.setType("os");
ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble);
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID());
***REMOVED***
***REMOVED***

17
src/test/java/ISClientTest.java
View File

@ -1,17 +0,0 @@
import java.net.ProxySelector;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class ISClientTest ***REMOVED***
public static void main(String[] args) ***REMOVED***
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(new ISClient().listDataminersInVRE());
***REMOVED***
***REMOVED***

119
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java
View File

@ -1,119 +0,0 @@
package org.gcube.dataanalysis.dataminerpoolmanager;
***REMOVED***
import java.io.InputStream;
import java.net.ProxySelector;
***REMOVED***
***REMOVED***
import java.util.Vector;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand;
import org.gcube.dataanalysis.dataminer.poolmanager.process.AlgorithmPackageParser;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class AlgorithmPackageParserTest ***REMOVED***
private static int BUFFER_SIZE = 2048;
public void extractAllAlgorithms() throws IOException ***REMOVED***
String url = "http:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms";
List<String> commands = this.extractAddAlgorithmCommands(url);
AlgorithmSet algorithms = new AlgorithmSet();
for (String cmd : commands) ***REMOVED***
System.out.println("-----------------------------------------");
System.out.println(cmd);
AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd);
System.out.println(aac);
***REMOVED*** start creating the algo from the command
Algorithm algo = new Algorithm();
algo.setAlgorithmType(aac.getAlgorithmType());
algo.setCategory(aac.getCategory());
algo.setClazz(aac.getClazz());
algo.setDescription(aac.getDescription());
algo.setName(aac.getName());
algo.setPackageURL(aac.getUrl());
algo.setSkipJava(aac.getSkipjava());
***REMOVED*** then override with info from the package
if (aac.getUrl().length() > 4) ***REMOVED***
Algorithm packagedAlgo = this.extractAlgorithm(aac.getUrl());
if (packagedAlgo != null) ***REMOVED***
algo.setDependencies(packagedAlgo.getDependencies());
***REMOVED***
***REMOVED***
algorithms.addAlgorithm(algo);
break;
***REMOVED***
***REMOVED***to uncomment
***REMOVED*** new DataminerPoolManager().addAlgorithmsToVRE(algorithms,
***REMOVED*** "/gcube/devNext/NextNext");
***REMOVED***
/**
* Extract 'addAlgorithm' commands from a file containing wiki-table-style
* entries for algorithm.
*
* @return
* @throws IOException
*/
private List<String> extractAddAlgorithmCommands(String listUrl)
throws IOException ***REMOVED***
URL url = new URL(listUrl);
InputStream is = url.openStream();
StringBuilder s = new StringBuilder();
byte[] buffer = new byte[BUFFER_SIZE];
int read = 0;
while ((read = is.read(buffer)) != -1) ***REMOVED***
s.append(new String(buffer, 0, read));
***REMOVED***
List<String> out = new Vector<>();
String[] lines = s.toString().split("\n");
for (String line : lines) ***REMOVED***
System.out.println("--------------------");
if (!line.isEmpty()) ***REMOVED***
String[] parts = line.split("\\|");
int c = 1;
for (String part : parts) ***REMOVED***
if (part == null || part.trim().isEmpty()) ***REMOVED***
continue;
***REMOVED***
System.out.println(c + ". " + part);
c++;
if (part.contains("addAlgorithm.sh")) ***REMOVED***
String cmd = part.trim();
cmd = cmd.replaceAll("<notextile>", "");
cmd = cmd.replaceAll("</notextile>", "");
System.out.println(cmd);
***REMOVED*** AddAlgorithmCommand aac = new AddAlgorithmCommand(cmd);
***REMOVED*** System.out.println(aac);
out.add(cmd);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
return out;
***REMOVED***
/**
* Create an Algorithm starting from the algorithm jar.
*
* @param url
* @return
* @throws IOException
*/
private Algorithm extractAlgorithm(String url) throws IOException ***REMOVED***
return new AlgorithmPackageParser().parsePackage(url);
***REMOVED***
public static void main(String[] args) throws Exception ***REMOVED***
***REMOVED*** ProxySelector.setDefault(new PropertiesBasedProxySelector(
***REMOVED*** "/home/ngalante/.proxy-settings"));
new AlgorithmPackageParserTest().extractAllAlgorithms();
***REMOVED***
***REMOVED***

33
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AnsibleWorkerTest.java
View File

@ -1,33 +0,0 @@
package org.gcube.dataanalysis.dataminerpoolmanager;
import java.io.File;
***REMOVED***
import java.util.UUID;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
public class AnsibleWorkerTest ***REMOVED***
public static void main(String[] args) throws IOException ***REMOVED***
AnsibleWorker worker = new AnsibleWorker(new File("/home/nagalante/gcube/dataminer-pool-manager/work/"+UUID.randomUUID().toString()));
System.out.println("created worker named " + worker.getWorkerId());
worker.setInventory(new Inventory());
worker.setPlaybook(new Playbook());
Role r = new Role();
r.setName("latex");
RoleFile tf = new RoleFile("main", "do something special for " + r.getName());
r.addTaskFile(tf);
worker.addRole(r);
worker.apply();
***REMOVED***
***REMOVED***

227
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java
View File

@ -1,227 +0,0 @@
package org.gcube.dataanalysis.dataminerpoolmanager;
import java.net.ProxySelector;
import java.util.UUID;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class DataminerPoolManagerTest ***REMOVED***
/*
private static Map<String, Domain> domains = new HashMap<>();
private static Map<String, Cluster> clusters = new HashMap<>();
private static Dependency createDependency(String depName) ***REMOVED***
String[] parts = depName.split(":");
Dependency out = new Dependency();
if(parts.length>1) ***REMOVED***
out.setType(parts[0]);
out.setName(parts[1]);
***REMOVED*** else ***REMOVED***
out.setType("os");
out.setName(depName);
***REMOVED***
return out;
***REMOVED***
private static Algorithm createAlgorithm(String name, String ... deps) ***REMOVED***
Algorithm a = new Algorithm();
a.setName(name);
for(String dep:deps) ***REMOVED***
a.addDependency(createDependency(dep));
***REMOVED***
return a;
***REMOVED***
private static AlgorithmSet createAlgorithmSet(String name, Algorithm ... algs) ***REMOVED***
AlgorithmSet out = new AlgorithmSet();
out.setName(name);
for(Algorithm a:algs) ***REMOVED***
out.addAlgorithm(a);
***REMOVED***
return out;
***REMOVED***
private static Domain getDomain(String name) ***REMOVED***
if(domains.get(name)==null) ***REMOVED***
Domain d = new Domain();
d.setName(name);
domains.put(name, d);
return d;
***REMOVED*** else ***REMOVED***
return domains.get(name);
***REMOVED***
***REMOVED***
private static Host createHost(String hostname, String domainName) ***REMOVED***
Host out = new Host();
out.setName(hostname);
Domain d = getDomain(domainName);
out.setDomain(d);
return out;
***REMOVED***
private static Cluster getCluster(String name) ***REMOVED***
if(clusters.get(name)==null) ***REMOVED***
Cluster d = new Cluster();
d.setName(name);
clusters.put(name, d);
return d;
***REMOVED*** else ***REMOVED***
return clusters.get(name);
***REMOVED***
***REMOVED***
private static Collection<Dependency> extractDependencies() ***REMOVED***
Collection<Dependency> out = new TreeSet<>(new DependencyComparator());
for(Cluster c:clusters.values()) ***REMOVED***
for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED***
for(Algorithm a:as.getAlgorithms()) ***REMOVED***
for(Dependency d:a.getDependencies()) ***REMOVED***
out.add(d);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
return out;
***REMOVED***
private static Collection<Algorithm> extractAlgorithms() ***REMOVED***
Collection<Algorithm> out = new TreeSet<>(new AlgorithmComparator());
for(Cluster c:clusters.values()) ***REMOVED***
for(AlgorithmSet as:c.getAlgorithmSets()) ***REMOVED***
for(Algorithm a:as.getAlgorithms()) ***REMOVED***
out.add(a);
***REMOVED***
***REMOVED***
***REMOVED***
return out;
***REMOVED***
static ***REMOVED***
Algorithm ewe = createAlgorithm("ewe", "mono", "latex", "cran:some_R_package", "custom:some_git_package");
Algorithm ensemble = createAlgorithm("ensemble", "python");
Algorithm voodoo = createAlgorithm("voodoo", "os:latex", "custom:blah");
AlgorithmSet as1 = createAlgorithmSet("as1-fishes", ewe);
AlgorithmSet as2 = createAlgorithmSet("as2-stat", ensemble);
AlgorithmSet as3 = createAlgorithmSet("as3-blackmagic", voodoo, ewe);
Cluster cluster1 = getCluster("cluster-1");
cluster1.addHost(createHost("host1", "domain1"));
cluster1.addHost(createHost("host2", "domain1"));
cluster1.addHost(createHost("host3", "domain1"));
cluster1.addHost(createHost("host1", "domain2"));
cluster1.addHost(createHost("host2", "domain2"));
Cluster cluster2 = getCluster("cluster-2");
cluster2.addHost(createHost("host4", "domain1"));
cluster2.addHost(createHost("host5", "domain1"));
cluster2.addHost(createHost("host6", "domain1"));
cluster2.addHost(createHost("host3", "domain2"));
cluster2.addHost(createHost("host4", "domain2"));
cluster2.addHost(createHost("host5", "domain2"));
cluster1.addAlgorithmSet(as1);
cluster1.addAlgorithmSet(as2);
cluster2.addAlgorithmSet(as1);
cluster2.addAlgorithmSet(as3);
***REMOVED***
*/
public static void main(String[] args) throws Exception ***REMOVED***
/*
AnsibleBridge ab = new AnsibleBridge();
ab.printInventoryByDomainAndSets(clusters.values());
System.out.println("-----------");
ab.printInventoryBySets(clusters.values());
AnsibleWorker worker = ab.createWorker();
for(Algorithm a:extractAlgorithms()) ***REMOVED***
for(Role r:ab.generateRoles(a)) ***REMOVED***
worker.addRole(r);
***REMOVED***
***REMOVED***
for(Dependency d:extractDependencies()) ***REMOVED***
for(Role r:ab.generateRoles(d)) ***REMOVED***
worker.addRole(r);
***REMOVED***
***REMOVED***
*/
ScopeProvider.instance.set("/gcube/devNext/NextNext");
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
***REMOVED***
***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc)
***REMOVED*** Algorithm algorithm = new Algorithm();
***REMOVED*** algorithm.setName("ichtyop");
***REMOVED*** algorithm.setClazz("org.gcube...");
***REMOVED*** algorithm.setDescription("some description");
***REMOVED***
***REMOVED*** Dependency d = new Dependency();
***REMOVED*** d.setName("libpng");
***REMOVED*** d.setType("os");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("some-r-package");
***REMOVED*** d.setType("cran");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("some-other-r-package");
***REMOVED*** d.setType("cran");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("voodoo");
***REMOVED*** d.setType("custom");
***REMOVED*** algorithm.addDependency(d);
***REMOVED***
***REMOVED*** ***REMOVED*** create the algorithm (download it, etc etc)
***REMOVED*** Algorithm ewe = new Algorithm();
***REMOVED*** ewe.setName("ewe");
***REMOVED***
***REMOVED*** d = new Dependency();
***REMOVED*** d.setName("voodoo");
***REMOVED*** d.setType("custom");
***REMOVED*** ewe.addDependency(d);
AlgorithmSet algorithms = new AlgorithmSet();
algorithms.setName("dummy-set");
***REMOVED*** algorithms.addAlgorithm(algorithm);
***REMOVED*** algorithms.addAlgorithm(ewe);
Algorithm ensemble = new Algorithm();
ensemble.setName("ensemble");
ensemble.setCategory("ICHTHYOP_MODEL");
ensemble.setAlgorithmType("transducerers");
ensemble.setPackageURL("http:***REMOVED***data.d4science.org/R0FqV2lNOW1jMkxuUEIrWXY4aUhvSENHSmVMQks4NjdHbWJQNStIS0N6Yz0");
ensemble.setClazz("org.gcube.dataanalysis.executor.rscripts.Ichthyopmodelonebyone");
ensemble.setSkipJava("N");
ensemble.setDescription("test");
Dependency d = new Dependency();
d.setName("libpng3");
d.setType("os");
ensemble.addDependency(d);
algorithms.addAlgorithm(ensemble);
new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID());
***REMOVED***
***REMOVED***

17
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/ISClientTest.java
View File

@ -1,17 +0,0 @@
package org.gcube.dataanalysis.dataminerpoolmanager;
import java.net.ProxySelector;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
public class ISClientTest ***REMOVED***
public static void main(String[] args) ***REMOVED***
ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(new ISClient().listDataminersInVRE());
***REMOVED***
***REMOVED***