gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@144486 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-03-01 15:21:40 +00:00
parent ae47b4745d
commit 4881cb6d16
4 changed files with 155 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -203,11 +203,12 @@ public class AnsibleBridge {
FileOutputStream fos = new FileOutputStream(n);
PrintStream ps = new PrintStream(fos);
System.setOut(ps);
System.setErr(ps);
worker.apply();
System.setOut(console);
//System.setErr(ps);
System.setErr(console);
worker.apply();
//System.setOut(console);
//worker.apply();
System.out.println("Log stored to to " + n.getAbsolutePath());
// destroy the worker
@ -235,6 +236,8 @@ public class AnsibleBridge {
*/
public Collection<Role> generateRoles(Dependency d) {
Collection<Role> roles = new Vector<>();
if("os".equalsIgnoreCase(d.getType())) {
OSDependencyPackage pkg = new OSDependencyPackage(d);
if(pkg!=null) {

3
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -30,7 +30,8 @@ public class ISClient {
Collection<Host> out = new Vector<>();
Host h = new Host();
//h.setName("bb-dataminer.res.eng.it");
h.setName("vm101.ui.savba.sk");
//h.setName("vm101.ui.savba.sk");
h.setName("dataminer1-devnext.d4science.org");
out.add(h);
return out;
} else {

103
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java
View File

@ -10,6 +10,7 @@ import java.util.Vector;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.lang.StringEscapeUtils;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
@ -149,24 +150,91 @@ public class AlgorithmPackageParser {
* @param metadata
* @return
*/
private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
Algorithm out = new Algorithm();
out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
List<String> dependencies = extractMultipleValues(metadata, METADATA_PACKAGES);
if (dependencies != null) {
for (String pkg : dependencies) {
Dependency dep = new Dependency();
dep.setName(pkg);
dep.setType("os");
out.addDependency(dep);
}
}
return out;
}
// private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
// Algorithm out = new Algorithm();
// out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
// out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
// out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
// List<String> dependencies = extractMultipleValues(metadata, METADATA_PACKAGES);
// if (dependencies != null) {
// for (String pkg : dependencies) {
// Dependency dep = new Dependency();
// dep.setName(pkg);
// dep.setType("os");
// out.addDependency(dep);
// }
// }
// return out;
// }
private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
Algorithm out = new Algorithm();
out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
List<String> rdependencies = extractMultipleValues(metadata, "r");
if (rdependencies != null) {
for (String pkg : rdependencies) {
Dependency dep = new Dependency();
//if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("cran");
out.addDependency(dep);
}
}
List<String> osdependencies = extractMultipleValues(metadata, "os");
if (osdependencies != null) {
for (String pkg : osdependencies) {
Dependency dep = new Dependency();
//if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("os");
out.addDependency(dep);
}
}
List<String> cdependencies = extractMultipleValues(metadata, "custom");
if (cdependencies != null) {
for (String pkg : cdependencies) {
Dependency dep = new Dependency();
//if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("custom");
out.addDependency(dep);
}
}
// if (pkg.startsWith("r:")){
// //String results = StringEscapeUtils.escapeJava(pkg);
// dep.setName(pkg);
// dep.setType("cran");
// }
// if (pkg.startsWith("custom:")){
// dep.setName(pkg);
// dep.setType("custom");
// }
// if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:")){
// dep.setName(pkg);
// dep.setType("os");
// }
return out;
}
private static String extractSingleValue(Map<String, List<String>> metadata,
String key) {
List<String> l = metadata.get(key);
@ -177,6 +245,9 @@ public class AlgorithmPackageParser {
}
}
private static List<String> extractMultipleValues(
Map<String, List<String>> metadata, String key) {
List<String> l = metadata.get(key);

99
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -1,8 +1,8 @@
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.ProxySelector;
import java.net.URL;
import java.net.UnknownHostException;
@ -13,12 +13,14 @@ import javax.ws.rs.QueryParam;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.gcube.dataanalysis.dataminerpoolmanager.AlgorithmPackageParserTest;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@Path("/")
public class RestPoolManager implements PoolManager {
@ -26,52 +28,46 @@ public class RestPoolManager implements PoolManager {
private PoolManager service = new DataminerPoolManager();
@GET
@Path("/add")
@Produces("text/plain")
public String addAlgorithmToVRE(@QueryParam("algorithm") String algorithm, @QueryParam("vre") String vre,
@QueryParam("name") String name, @QueryParam("description") String description,
@QueryParam("category") String category, @QueryParam("algorithmType") String algorithmType,
public String addAlgorithmToVRE(
@QueryParam("algorithm") String algorithm,
@QueryParam("vre") String vre,
@QueryParam("name") String name,
@QueryParam("description") String description,
@QueryParam("category") String category,
@QueryParam("algorithmType") String algorithmType,
@QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
// TODO Auto-generated method stub
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
Algorithm algo = service.extractAlgorithm(algorithm);
if (algo.getCategory() == null) {
if (algo.getCategory() == null){
algo.setCategory(category);
} else
category = algo.getCategory();
System.out.println("category is: " + category);
if (algo.getAlgorithmType() == null) {
} else category = algo.getCategory();
if (algo.getAlgorithmType() == null){
algo.setAlgorithmType(algorithmType);
} else
algorithmType = algo.getCategory();
System.out.println("algorithmType is: " + algorithmType);
if (algo.getSkipJava() == null) {
} else algorithmType = algo.getCategory();
if (algo.getSkipJava() == null){
algo.setSkipJava(skipJava);
} else
skipJava = algo.getSkipJava();
System.out.println("skipJava is: " + skipJava);
if (algo.getName() == null) {
} else skipJava = algo.getSkipJava();
if (algo.getName() == null){
algo.setCategory(name);
} else
name = algo.getName();
System.out.println("name is: " + name);
if (algo.getDescription() == null) {
algo.setDescription(description);
;
} else
description = algo.getDescription();
System.out.println("description is: " + description);
} else name = algo.getName();
if (algo.getDescription() == null){
algo.setDescription(description);;
} else description = algo.getDescription();
return service.addAlgorithmToVRE(algo, vre);
}
@GET
@Path("/log")
@Produces("text/plain")
@ -80,6 +76,10 @@ public class RestPoolManager implements PoolManager {
LOGGER.debug("Returning Log =" + logUrl);
return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl));
}
@Override
public Algorithm extractAlgorithm(String url) throws IOException {
@ -87,37 +87,58 @@ public class RestPoolManager implements PoolManager {
return null;
}
public static void main(String[] args) throws IOException, InterruptedException {
RestPoolManager a = new RestPoolManager();
// System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
//System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
//ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
ScopeProvider.instance.set("/gcube/devNext/NextNext");
a.addAlgorithmToVRE("http://data.d4science.org/am84enRYYkZ3dGRjZk9qTytQTndqaFpTOFJtT3RQeGhHbWJQNStIS0N6Yz0",
"/gcube/devNext/NextNext", null, null, "ICHTHYOP_MODEL", "transducerers", "N");
// System.out.println(a.getLogById("dadcb059-69e5-48c3-aa58-3b290ae0419d"));
a.addAlgorithmToVRE(
"http://data.d4science.org/cXhJV0s4b3dUN05jZk9qTytQTndqdmVUdURLOTRHeHpHbWJQNStIS0N6Yz0",
"/gcube/devNext/NextNext",
null, null, "test", "transducerers", "N");
//System.out.println(a.getLogById("34ac474d-b9df-4929-87e1-2a0ae26cf898"));
}
@Override
public void getLogId(Algorithm algo, String vre) {
// TODO Auto-generated method stub
}
@Override
public String getScriptFromURL(URL logId) throws IOException {
// TODO Auto-generated method stub
return null;
}
@Override
public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException {
// TODO Auto-generated method stub
return null;
}
@Override
public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException {
// TODO Auto-generated method stub
return null;
}
}