gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Feature/24026 #3

Merged
giancarlo.panichi merged 194 commits from Feature/24026 into master 2023-11-09 19:12:54 +01:00
Conversation 0 Commits 194 Files Changed 7 +155 -59
4 changed files with 155 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit c40cec3733 - Show all commits

9
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -203,11 +203,12 @@ public class AnsibleBridge {
FileOutputStream fos = new FileOutputStream(n);
PrintStream ps = new PrintStream(fos);
System.setOut(ps);
System.setErr(ps);
worker.apply();
System.setOut(console);
***REMOVED***System.setErr(ps);
System.setErr(console);
worker.apply();
***REMOVED***System.setOut(console);
***REMOVED***worker.apply();
System.out.println("Log stored to to " + n.getAbsolutePath());
***REMOVED*** destroy the worker
@ -235,6 +236,8 @@ public class AnsibleBridge {
*/
public Collection<Role> generateRoles(Dependency d) {
Collection<Role> roles = new Vector<>();
if("os".equalsIgnoreCase(d.getType())) {
OSDependencyPackage pkg = new OSDependencyPackage(d);
if(pkg!=null) {

3
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -30,7 +30,8 @@ public class ISClient {
Collection<Host> out = new Vector<>();
Host h = new Host();
***REMOVED***h.setName("bb-dataminer.res.eng.it");
h.setName("vm101.ui.savba.sk");
***REMOVED***h.setName("vm101.ui.savba.sk");
h.setName("dataminer1-devnext.d4science.org");
out.add(h);
return out;
***REMOVED*** else {

103
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java
View File

@ -10,6 +10,7 @@ import java.util.Vector;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import org.apache.commons.lang.StringEscapeUtils;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
@ -149,24 +150,91 @@ public class AlgorithmPackageParser {
* @param metadata
* @return
*/
private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
Algorithm out = new Algorithm();
out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
List<String> dependencies = extractMultipleValues(metadata, METADATA_PACKAGES);
if (dependencies != null) {
for (String pkg : dependencies) {
Dependency dep = new Dependency();
dep.setName(pkg);
dep.setType("os");
out.addDependency(dep);
***REMOVED***
***REMOVED***
return out;
***REMOVED***
***REMOVED*** private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
***REMOVED*** Algorithm out = new Algorithm();
***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
***REMOVED*** out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
***REMOVED*** out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
***REMOVED*** List<String> dependencies = extractMultipleValues(metadata, METADATA_PACKAGES);
***REMOVED*** if (dependencies != null) {
***REMOVED*** for (String pkg : dependencies) {
***REMOVED*** Dependency dep = new Dependency();
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("os");
***REMOVED*** out.addDependency(dep);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** return out;
***REMOVED*** ***REMOVED***
private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
Algorithm out = new Algorithm();
out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
out.setDescription(extractSingleValue(metadata, METADATA_ALGORITHM_DESCRIPTION));
out.setClazz(extractSingleValue(metadata, METADATA_CLASS_NAME));
List<String> rdependencies = extractMultipleValues(metadata, "r");
if (rdependencies != null) {
for (String pkg : rdependencies) {
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("cran");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> osdependencies = extractMultipleValues(metadata, "os");
if (osdependencies != null) {
for (String pkg : osdependencies) {
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("os");
out.addDependency(dep);
***REMOVED***
***REMOVED***
List<String> cdependencies = extractMultipleValues(metadata, "custom");
if (cdependencies != null) {
for (String pkg : cdependencies) {
Dependency dep = new Dependency();
***REMOVED***if (pkg.startsWith("os:")){
dep.setName(pkg);
dep.setType("custom");
out.addDependency(dep);
***REMOVED***
***REMOVED***
***REMOVED*** if (pkg.startsWith("r:")){
***REMOVED*** ***REMOVED***String results = StringEscapeUtils.escapeJava(pkg);
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("cran");
***REMOVED*** ***REMOVED***
***REMOVED*** if (pkg.startsWith("custom:")){
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("custom");
***REMOVED*** ***REMOVED***
***REMOVED*** if (!pkg.startsWith("os:")&&!pkg.startsWith("r:")&&!pkg.startsWith("custom:")){
***REMOVED*** dep.setName(pkg);
***REMOVED*** dep.setType("os");
***REMOVED*** ***REMOVED***
return out;
***REMOVED***
private static String extractSingleValue(Map<String, List<String>> metadata,
String key) {
List<String> l = metadata.get(key);
@ -177,6 +245,9 @@ public class AlgorithmPackageParser {
***REMOVED***
***REMOVED***
private static List<String> extractMultipleValues(
Map<String, List<String>> metadata, String key) {
List<String> l = metadata.get(key);

99
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -1,8 +1,8 @@
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
import java.io.File;
***REMOVED***
import java.net.MalformedURLException;
import java.net.ProxySelector;
***REMOVED***
import java.net.UnknownHostException;
@ -13,12 +13,14 @@ import javax.ws.rs.QueryParam;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.process.AddAlgorithmCommand;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.gcube.dataanalysis.dataminerpoolmanager.AlgorithmPackageParserTest;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
***REMOVED***
***REMOVED***
@Path("/")
public class RestPoolManager implements PoolManager {
@ -26,52 +28,46 @@ public class RestPoolManager implements PoolManager {
private PoolManager service = new DataminerPoolManager();
@GET
@Path("/add")
@Produces("text/plain")
public String addAlgorithmToVRE(@QueryParam("algorithm") String algorithm, @QueryParam("vre") String vre,
@QueryParam("name") String name, @QueryParam("description") String description,
@QueryParam("category") String category, @QueryParam("algorithmType") String algorithmType,
public String addAlgorithmToVRE(
@QueryParam("algorithm") String algorithm,
@QueryParam("vre") String vre,
@QueryParam("name") String name,
@QueryParam("description") String description,
@QueryParam("category") String category,
@QueryParam("algorithmType") String algorithmType,
@QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
***REMOVED*** TODO Auto-generated method stub
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
Algorithm algo = service.extractAlgorithm(algorithm);
if (algo.getCategory() == null) {
if (algo.getCategory() == null){
algo.setCategory(category);
***REMOVED*** else
category = algo.getCategory();
System.out.println("category is: " + category);
if (algo.getAlgorithmType() == null) {
***REMOVED*** else category = algo.getCategory();
if (algo.getAlgorithmType() == null){
algo.setAlgorithmType(algorithmType);
***REMOVED*** else
algorithmType = algo.getCategory();
System.out.println("algorithmType is: " + algorithmType);
if (algo.getSkipJava() == null) {
***REMOVED*** else algorithmType = algo.getCategory();
if (algo.getSkipJava() == null){
algo.setSkipJava(skipJava);
***REMOVED*** else
skipJava = algo.getSkipJava();
System.out.println("skipJava is: " + skipJava);
if (algo.getName() == null) {
***REMOVED*** else skipJava = algo.getSkipJava();
if (algo.getName() == null){
algo.setCategory(name);
***REMOVED*** else
name = algo.getName();
System.out.println("name is: " + name);
if (algo.getDescription() == null) {
algo.setDescription(description);
;
***REMOVED*** else
description = algo.getDescription();
System.out.println("description is: " + description);
***REMOVED*** else name = algo.getName();
if (algo.getDescription() == null){
algo.setDescription(description);;
***REMOVED*** else description = algo.getDescription();
return service.addAlgorithmToVRE(algo, vre);
***REMOVED***
@GET
@Path("/log")
@Produces("text/plain")
@ -80,6 +76,10 @@ public class RestPoolManager implements PoolManager {
LOGGER.debug("Returning Log =" + logUrl);
return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl));
***REMOVED***
@Override
public Algorithm extractAlgorithm(String url) throws IOException {
@ -87,37 +87,58 @@ public class RestPoolManager implements PoolManager {
return null;
***REMOVED***
public static void main(String[] args) throws IOException, InterruptedException {
RestPoolManager a = new RestPoolManager();
***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
ScopeProvider.instance.set("/gcube/devNext/NextNext");
a.addAlgorithmToVRE("http:***REMOVED***data.d4science.org/am84enRYYkZ3dGRjZk9qTytQTndqaFpTOFJtT3RQeGhHbWJQNStIS0N6Yz0",
"/gcube/devNext/NextNext", null, null, "ICHTHYOP_MODEL", "transducerers", "N");
***REMOVED*** System.out.println(a.getLogById("dadcb059-69e5-48c3-aa58-3b290ae0419d"));
a.addAlgorithmToVRE(
"http:***REMOVED***data.d4science.org/cXhJV0s4b3dUN05jZk9qTytQTndqdmVUdURLOTRHeHpHbWJQNStIS0N6Yz0",
"/gcube/devNext/NextNext",
null, null, "test", "transducerers", "N");
***REMOVED***System.out.println(a.getLogById("34ac474d-b9df-4929-87e1-2a0ae26cf898"));
***REMOVED***
@Override
public void getLogId(Algorithm algo, String vre) {
***REMOVED*** TODO Auto-generated method stub
***REMOVED***
@Override
public String getScriptFromURL(URL logId) throws IOException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
@Override
public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
@Override
public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException {
***REMOVED*** TODO Auto-generated method stub
return null;
***REMOVED***
***REMOVED***