From 2189ab14b4690b78a7c036968d79bb4fcd8acf2c Mon Sep 17 00:00:00 2001 From: Paolo Fabriani Date: Wed, 23 Nov 2016 08:42:15 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@134567 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../template/AlgorithmPackage.java | 2 +- .../template/CustomDependencyPackage.java | 8 +- .../template/DependencyPackage.java | 2 +- .../datamodel/Algorithm.java | 11 + .../process/AddAlgorithmCommand.java | 90 +++++++++ .../process/AlgorithmPackageParser.java | 190 ++++++++++++++++++ 6 files changed, 295 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AddAlgorithmCommand.java create mode 100644 src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AlgorithmPackageParser.java diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java index 5870a28..8bd958f 100644 --- a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/AlgorithmPackage.java @@ -37,7 +37,7 @@ public class AlgorithmPackage { public Collection getRoles(TemplateManager tm) { Collection out = new Vector<>(); - for(String mode:new String[]{"add", "remove", "update"***REMOVED***) { + for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" String roleName = "gcube-algorithm-"+this.getAlgorithm().getName()+("add".equals(mode) ? "" : "-"+mode); try { ***REMOVED*** find template diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java index 47464e0..4818389 100644 --- a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/CustomDependencyPackage.java @@ -1,15 +1,10 @@ package org.gcube.dataanalysys.dataminerpoolmanager.ansiblebridge.template; -import java.io.File; -***REMOVED*** import java.util.Collection; -import java.util.Map; import java.util.NoSuchElementException; import java.util.Vector; -import org.apache.commons.io.FileUtils; import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.Role; -import org.gcube.dataanalysys.dataminerpoolmanager.ansible.model.RoleFile; import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; public class CustomDependencyPackage extends DependencyPackage { @@ -47,7 +42,8 @@ public class CustomDependencyPackage extends DependencyPackage { public Collection getRoles(CustomRoleManager crm) { Collection out = new Vector<>(); - for(String mode:new String[]{"add", "remove", "update"***REMOVED***) { +***REMOVED*** for(String mode:new String[]{"add", "remove", "update"***REMOVED***) { + for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" ***REMOVED*** role name String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); try { diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java index d8b7dd8..abd59dd 100644 --- a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/ansiblebridge/template/DependencyPackage.java @@ -31,7 +31,7 @@ public class DependencyPackage { public Collection getRoles(TemplateManager tm) { Collection out = new Vector<>(); - for(String mode:new String[]{"add", "remove", "update"***REMOVED***) { + for(String mode:new String[]{"add"***REMOVED***) { ***REMOVED*** "remove", "update" String roleName = this.getDependency().getType()+"-"+this.getDependency().getName()+("add".equals(mode) ? "" : "-"+mode); try { ***REMOVED*** find template diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java index cbbb912..98f3eb7 100644 --- a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/datamodel/Algorithm.java @@ -10,6 +10,8 @@ public class Algorithm { private String description; private String category; + + private String clazz; private Collection actions; @@ -63,9 +65,18 @@ public class Algorithm { public String toString() { String out = "Algorithm: " + this.getName()+"\n"; + out+=" Class Name: " + this.getClazz()+"\n"; out+=" Description: " + this.getDescription()+"\n"; out+=" Dependencies: " + this.getDependencies()+"\n"; return out; ***REMOVED*** + public String getClazz() { + return clazz; + ***REMOVED*** + + public void setClazz(String clazz) { + this.clazz = clazz; + ***REMOVED*** + ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AddAlgorithmCommand.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AddAlgorithmCommand.java new file mode 100644 index 0000000..ffc4cdc --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AddAlgorithmCommand.java @@ -0,0 +1,90 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.process; + +import java.util.StringTokenizer; + +public class AddAlgorithmCommand { + + private String command; + private String name; + private String category; + private String clazz; + private String vre; + private String p6; + private String p7; + private String url; + private String description; + + public AddAlgorithmCommand(String cmd) { + StringTokenizer st = new StringTokenizer(cmd, " "); + if (st.hasMoreElements()) + command = st.nextToken(); + if (st.hasMoreElements()) + name = st.nextToken(); + if (st.hasMoreElements()) + category = st.nextToken(); + if (st.hasMoreElements()) + clazz = st.nextToken(); + if (st.hasMoreElements()) + vre = st.nextToken(); + if (st.hasMoreElements()) + p6 = st.nextToken(); + if (st.hasMoreElements()) + p7 = st.nextToken(); + if (st.hasMoreElements()) + url = st.nextToken(); + description = ""; + while (st.hasMoreElements()) + description = description + st.nextToken() + " "; + ***REMOVED*** + + public String getCommand() { + return command; + ***REMOVED*** + + public String getName() { + return name; + ***REMOVED*** + + public String getCategory() { + return category; + ***REMOVED*** + + public String getClazz() { + return clazz; + ***REMOVED*** + + public String getVRE() { + return vre; + ***REMOVED*** + + public String getP6() { + return p6; + ***REMOVED*** + + public String getP7() { + return p7; + ***REMOVED*** + + public String getUrl() { + return url; + ***REMOVED*** + + public String getDescription() { + return description; + ***REMOVED*** + + public String toString() { + String out = ""; + out += String.format("%-12s: %s\n", "command", command); + out += String.format("%-12s: %s\n", "name", name); + out += String.format("%-12s: %s\n", "category", category); + out += String.format("%-12s: %s\n", "class", clazz); + out += String.format("%-12s: %s\n", "vre", vre); + out += String.format("%-12s: %s\n", "p6", p6); + out += String.format("%-12s: %s\n", "p7", p7); + out += String.format("%-12s: %s\n", "url", url); + out += String.format("%-12s: %s\n", "description", this.description); + return out; + ***REMOVED*** + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AlgorithmPackageParser.java new file mode 100644 index 0000000..121ca80 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysys/dataminerpoolmanager/process/AlgorithmPackageParser.java @@ -0,0 +1,190 @@ +package org.gcube.dataanalysys.dataminerpoolmanager.process; + +***REMOVED*** +import java.io.InputStream; +***REMOVED*** +import java.util.HashMap; +***REMOVED*** +import java.util.Map; +import java.util.Vector; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Algorithm; +import org.gcube.dataanalysys.dataminerpoolmanager.datamodel.Dependency; + +public class AlgorithmPackageParser { + + /** + * The name of the file containing algorithm metadata. Expected in the root + * directory of the package. + */ + private static final String METADATA_FILE_NAME = "Info.txt"; + + private static final String METADATA_ALGORITHM_NAME = "Algorithm Name"; + + private static final String METADATA_ALGORITHM_DESCRIPTION = "Algorithm Description"; + + private static final String METADATA_CLASS_NAME = "Class Name"; + + private static final String METADATA_PACKAGES = "Packages"; + + private static final String METADATA_KEY_VALUE_SEPARATOR = ":"; + + private static final int BUFFER_SIZE = 4096; + + /** + * Given an URL to an algorithm package, create an Algorithm object with its + * metadata. Metadata are extracted from the 'info.txt' file, if any, in the + * package. + * + * @param url + * @return An Algorithm object or null if no 'info.txt' is found in the + * package. + * @throws IOException + */ + public Algorithm parsePackage(String url) throws IOException { + String packageMetadata = this.getPackageMetadata(url); + if (packageMetadata == null) { + System.out.println("WARNING: No metadata found for " + url); + return null; + ***REMOVED*** else { + Map> parsedMetadata = this.parseMetadata(packageMetadata); + return this.createAlgorithm(parsedMetadata); + ***REMOVED*** + ***REMOVED*** + + /** + * Extract the content of the metadata file from the package. + * + * @param url + * @return + * @throws IOException + */ + private String getPackageMetadata(String url) throws IOException { + InputStream is = new URL(url).openStream(); + ZipInputStream zipIs = new ZipInputStream(is); + ZipEntry entry = zipIs.getNextEntry(); + String out = null; + while (entry != null) { + if (METADATA_FILE_NAME.equalsIgnoreCase(entry.getName())) { + out = this.getEntryContent(zipIs); + break; + ***REMOVED*** + entry = zipIs.getNextEntry(); + ***REMOVED*** + is.close(); + zipIs.close(); + return out; + ***REMOVED*** + + /** + * Read the content of a zip entry and place it in a string. + * @param zipIn + * @return + * @throws IOException + */ + private String getEntryContent(ZipInputStream zipIn) throws IOException { + StringBuilder s = new StringBuilder(); + byte[] buffer = new byte[BUFFER_SIZE]; + int read = 0; + while ((read = zipIn.read(buffer)) != -1) { + s.append(new String(buffer, 0, read)); + ***REMOVED*** + return s.toString(); + ***REMOVED*** + + /** + * Parse the content of the metadata file and create a key+multivalue map. + * @param metadata + * @return + */ + private Map> parseMetadata(String metadata) { + Map> out = new HashMap>(); + String[] lines = metadata.split("\n"); + + String key = null; + String value = null; + + for (String line : lines) { + ***REMOVED*** skip empty lines + if (line.trim().isEmpty()) { + continue; + ***REMOVED*** + ***REMOVED*** scan lines one by one, looking for key and values + String[] parts = line.split(METADATA_KEY_VALUE_SEPARATOR); + if (parts.length > 1) { + ***REMOVED*** key and value on the same line + key = parts[0].trim(); + value = line.substring(parts[0].length() + 1).trim(); + ***REMOVED*** else if (parts.length == 1) { + ***REMOVED*** either a key or a value + if (line.trim().endsWith(METADATA_KEY_VALUE_SEPARATOR)) { + ***REMOVED*** key + key = parts[0].trim(); + value = null; + ***REMOVED*** else { + ***REMOVED*** value + value = line.trim(); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** add key+value to the map + if (key != null && value != null) { + List values = out.get(key); + if (values == null) { + values = new Vector<>(); + out.put(key, values); + ***REMOVED*** + values.add(value); + System.out.println(key + METADATA_KEY_VALUE_SEPARATOR + " " + values); + ***REMOVED*** + ***REMOVED*** + return out; + ***REMOVED*** + + /** + * Create an Algorithm starting from its metadata + * @param metadata + * @return + */ + private Algorithm createAlgorithm(Map> metadata) { + Algorithm out = new Algorithm(); + out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); + out.setDescription(extractSingleValue(metadata, + METADATA_ALGORITHM_DESCRIPTION)); + out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME)); + List dependencies = extractMultipleValues(metadata, + METADATA_PACKAGES); + if (dependencies != null) { + for (String pkg : dependencies) { + Dependency dep = new Dependency(); + dep.setName(pkg); + dep.setType("OS"); + out.addDependency(dep); + ***REMOVED*** + ***REMOVED*** + return out; + ***REMOVED*** + + + private static String extractSingleValue(Map> metadata, + String key) { + List l = metadata.get(key); + if (l != null && l.size() == 1) { + return l.get(0); + ***REMOVED*** else { + return null; + ***REMOVED*** + ***REMOVED*** + + private static List extractMultipleValues( + Map> metadata, String key) { + List l = metadata.get(key); + if (l != null) { + return new Vector<>(l); + ***REMOVED*** else { + return null; + ***REMOVED*** + ***REMOVED*** + +***REMOVED***