gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@134567 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Paolo Fabriani 2016-11-23 08:42:15 +00:00
parent 21313f0e53
commit 2189ab14b4
6 changed files with 295 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java
View File

@ -37,7 +37,7 @@ public class AlgorithmPackage {
public Collection<Role> getRoles(TemplateManager tm) {
Collection<Role> out = new Vector<>();
for(String mode:new String[]{"add", "remove", "update"***REMOVED***) {
for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update"
String roleName = "gcube-algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode);
try {
***REMOVED*** find template

8
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java
View File

@ -1,15 +1,10 @@
package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template;
import java.io.File;
***REMOVED***
import java.util.Collection;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Vector;
import org.apache.commons.io.FileUtils;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role;
import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class CustomDependencyPackage extends DependencyPackage {
@ -47,7 +42,8 @@ public class CustomDependencyPackage extends DependencyPackage {
public Collection<Role> getRoles(CustomRoleManager crm) {
Collection<Role> out = new Vector<>();
for(String mode:new String[]{"add", "remove", "update"***REMOVED***) {
***REMOVED*** for(String mode:new String[]{"add", "remove", "update"***REMOVED***) {
for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update"
***REMOVED*** role name
String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
try {

2
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java
View File

@ -31,7 +31,7 @@ public class DependencyPackage {
public Collection<Role> getRoles(TemplateManager tm) {
Collection<Role> out = new Vector<>();
for(String mode:new String[]{"add", "remove", "update"***REMOVED***) {
for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update"
String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode);
try {
***REMOVED*** find template

11
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java
View File

@ -10,6 +10,8 @@ public class Algorithm {
private String description;
private String category;
private String clazz;
private Collection<Action> actions;
@ -63,9 +65,18 @@ public class Algorithm {
public String toString() {
String out = "Algorithm: " + this.getName()+"\n";
out+=" Class Name: " + this.getClazz()+"\n";
out+=" Description: " + this.getDescription()+"\n";
out+=" Dependencies: " + this.getDependencies()+"\n";
return out;
***REMOVED***
public String getClazz() {
return clazz;
***REMOVED***
public void setClazz(String clazz) {
this.clazz = clazz;
***REMOVED***
***REMOVED***

90
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AddAlgorithmCommand.java Normal file
View File

@ -0,0 +1,90 @@
package org.gcube.dataanalysys.dataminerpoolmanager.process;
import java.util.StringTokenizer;
public class AddAlgorithmCommand {
private String command;
private String name;
private String category;
private String clazz;
private String vre;
private String p6;
private String p7;
private String url;
private String description;
public AddAlgorithmCommand(String cmd) {
StringTokenizer st = new StringTokenizer(cmd, " ");
if (st.hasMoreElements())
command = st.nextToken();
if (st.hasMoreElements())
name = st.nextToken();
if (st.hasMoreElements())
category = st.nextToken();
if (st.hasMoreElements())
clazz = st.nextToken();
if (st.hasMoreElements())
vre = st.nextToken();
if (st.hasMoreElements())
p6 = st.nextToken();
if (st.hasMoreElements())
p7 = st.nextToken();
if (st.hasMoreElements())
url = st.nextToken();
description = "";
while (st.hasMoreElements())
description = description + st.nextToken() + " ";
***REMOVED***
public String getCommand() {
return command;
***REMOVED***
public String getName() {
return name;
***REMOVED***
public String getCategory() {
return category;
***REMOVED***
public String getClazz() {
return clazz;
***REMOVED***
public String getVRE() {
return vre;
***REMOVED***
public String getP6() {
return p6;
***REMOVED***
public String getP7() {
return p7;
***REMOVED***
public String getUrl() {
return url;
***REMOVED***
public String getDescription() {
return description;
***REMOVED***
public String toString() {
String out = "";
out += String.format("%-12s: %s\n", "command", command);
out += String.format("%-12s: %s\n", "name", name);
out += String.format("%-12s: %s\n", "category", category);
out += String.format("%-12s: %s\n", "class", clazz);
out += String.format("%-12s: %s\n", "vre", vre);
out += String.format("%-12s: %s\n", "p6", p6);
out += String.format("%-12s: %s\n", "p7", p7);
out += String.format("%-12s: %s\n", "url", url);
out += String.format("%-12s: %s\n", "description", this.description);
return out;
***REMOVED***
***REMOVED***

190
src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AlgorithmPackageParser.java Normal file
View File

@ -0,0 +1,190 @@
package org.gcube.dataanalysys.dataminerpoolmanager.process;
***REMOVED***
import java.io.InputStream;
***REMOVED***
import java.util.HashMap;
***REMOVED***
import java.util.Map;
import java.util.Vector;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm;
import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency;
public class AlgorithmPackageParser {
/**
* The name of the file containing algorithm metadata. Expected in the root
* directory of the package.
*/
private static final String METADATA_FILE_NAME = "Info.txt";
private static final String METADATA_ALGORITHM_NAME = "Algorithm Name";
private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description";
private static final String METADATA_CLASS_NAME = "Class Name";
private static final String METADATA_PACKAGES = "Packages";
private static final String METADATA_KEY_VALUE_SEPARATOR = ":";
private static final int BUFFER_SIZE = 4096;
/**
* Given an URL to an algorithm package, create an Algorithm object with its
* metadata. Metadata are extracted from the 'info.txt' file, if any, in the
* package.
*
* @param url
* @return An Algorithm object or null if no 'info.txt' is found in the
* package.
* @throws IOException
*/
public Algorithm parsePackage(String url) throws IOException {
String packageMetadata = this.getPackageMetadata(url);
if (packageMetadata == null) {
System.out.println("WARNING: No metadata found for " + url);
return null;
***REMOVED*** else {
Map<String, List<String>> parsedMetadata = this.parseMetadata(packageMetadata);
return this.createAlgorithm(parsedMetadata);
***REMOVED***
***REMOVED***
/**
* Extract the content of the metadata file from the package.
*
* @param url
* @return
* @throws IOException
*/
private String getPackageMetadata(String url) throws IOException {
InputStream is = new URL(url).openStream();
ZipInputStream zipIs = new ZipInputStream(is);
ZipEntry entry = zipIs.getNextEntry();
String out = null;
while (entry != null) {
if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) {
out = this.getEntryContent(zipIs);
break;
***REMOVED***
entry = zipIs.getNextEntry();
***REMOVED***
is.close();
zipIs.close();
return out;
***REMOVED***
/**
* Read the content of a zip entry and place it in a string.
* @param zipIn
* @return
* @throws IOException
*/
private String getEntryContent(ZipInputStream zipIn) throws IOException {
StringBuilder s = new StringBuilder();
byte[] buffer = new byte[BUFFER_SIZE];
int read = 0;
while ((read = zipIn.read(buffer)) != -1) {
s.append(new String(buffer, 0, read));
***REMOVED***
return s.toString();
***REMOVED***
/**
* Parse the content of the metadata file and create a key+multivalue map.
* @param metadata
* @return
*/
private Map<String, List<String>> parseMetadata(String metadata) {
Map<String, List<String>> out = new HashMap<String, List<String>>();
String[] lines = metadata.split("\n");
String key = null;
String value = null;
for (String line : lines) {
***REMOVED*** skip empty lines
if (line.trim().isEmpty()) {
continue;
***REMOVED***
***REMOVED*** scan lines one by one, looking for key and values
String[] parts = line.split(METADATA_KEY_VALUE_SEPARATOR);
if (parts.length > 1) {
***REMOVED*** key and value on the same line
key = parts[0].trim();
value = line.substring(parts[0].length() + 1).trim();
***REMOVED*** else if (parts.length == 1) {
***REMOVED*** either a key or a value
if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) {
***REMOVED*** key
key = parts[0].trim();
value = null;
***REMOVED*** else {
***REMOVED*** value
value = line.trim();
***REMOVED***
***REMOVED***
***REMOVED*** add key+value to the map
if (key != null && value != null) {
List<String> values = out.get(key);
if (values == null) {
values = new Vector<>();
out.put(key, values);
***REMOVED***
values.add(value);
System.out.println(key + METADATA_KEY_VALUE_SEPARATOR + " " + values);
***REMOVED***
***REMOVED***
return out;
***REMOVED***
/**
* Create an Algorithm starting from its metadata
* @param metadata
* @return
*/
private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
Algorithm out = new Algorithm();
out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
out.setDescription(extractSingleValue(metadata,
METADATA_ALGORITHM_DESCRIPTION));
out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
List<String> dependencies = extractMultipleValues(metadata,
METADATA_PACKAGES);
if (dependencies != null) {
for (String pkg : dependencies) {
Dependency dep = new Dependency();
dep.setName(pkg);
dep.setType("OS");
out.addDependency(dep);
***REMOVED***
***REMOVED***
return out;
***REMOVED***
private static String extractSingleValue(Map<String, List<String>> metadata,
String key) {
List<String> l = metadata.get(key);
if (l != null && l.size() == 1) {
return l.get(0);
***REMOVED*** else {
return null;
***REMOVED***
***REMOVED***
private static List<String> extractMultipleValues(
Map<String, List<String>> metadata, String key) {
List<String> l = metadata.get(key);
if (l != null) {
return new Vector<>(l);
***REMOVED*** else {
return null;
***REMOVED***
***REMOVED***
***REMOVED***