gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Merge pull request 'Feature/24026' (!3) from Feature/24026 into master 

Reviewed-on: #3
This commit is contained in:
Giancarlo Panichi 2023-11-09 19:12:52 +01:00
parent 4a5eb67f73 033ce0b37b
commit e1d88418be
7 changed files with 86 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -1,7 +1,7 @@
# Changelog
## [v2.7.2-SNAPSHOT]
## [v2.8.0-SNAPSHOT]
- Updated to gcube-smartgears-bom.2.5.0

2
changelog.xml
View File

@ -1,5 +1,5 @@
<ReleaseNotes>
<Changeset component="org.gcube.dataanalysis.dataminer-pool-manager.2-7-2" date="2023-11-09">
<Changeset component="org.gcube.dataanalysis.dataminer-pool-manager.2-8-0" date="2023-11-09">
<Change>Updated to gcube-smartgears-bom.2.5.0</Change>
</Changeset>
<Changeset component="org.gcube.dataanalysis.dataminer-pool-manager.2-7-1" date="2022-04-06">

2
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataanalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<packaging>war</packaging>
<version>2.7.2-SNAPSHOT</version>
<version>2.8.0-SNAPSHOT</version>
<name>dataminer-pool-manager</name>
<description>DataMiner Pool Manager is a service to support the integration of algorithms in D4Science Infrastructure</description>

9
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java
View File

@ -20,6 +20,9 @@ import au.com.bytecode.opencsv.CSVReader;
public class HAProxy {
private static final String token1 = "";
private static final String token2 = "";
private Logger logger;
private CSVReader reader;
@ -142,16 +145,16 @@ public class HAProxy {
HAProxy a = new HAProxy();
//ScopeProvider.instance.set("/gcube/devNext/NextNext");
//ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SecurityTokenProvider.instance.set("");
SecurityTokenProvider.instance.set(token1);
CheckPermission test = new CheckPermission();
CheckPermission.apply("", "/gcube/devNext/NextNext");
CheckPermission.apply(token2, "/gcube/devNext/NextNext");
//ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
// System.out.println(a.getHProxy());
// System.out.println(a.MapCluster());
//System.out.println(a.listDataMinersByCluster("","/gcube/devNext/NextNext"));
//System.out.println(a.listDataMinersByCluster("xxx-xxx-xxx-xxx","/gcube/devNext/NextNext"));
// System.out.println(a.listDataMinersByCluster());
// List<Dependency> list = new LinkedList<Dependency>();

161
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -1,6 +1,5 @@
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
@ -24,52 +23,44 @@ import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNException;
@Path("/")
public class RestPoolManager implements PoolManager
{
//@Context ServletContext context;
public class RestPoolManager implements PoolManager {
// @Context ServletContext context;
private final Logger logger;
private DataminerPoolManager service;
//@Context
//private ApplicationContext context = ContextProvider.get();
public RestPoolManager()
{
// @Context
// private ApplicationContext context = ContextProvider.get();
public RestPoolManager() {
this.logger = LoggerFactory.getLogger(RestPoolManager.class);
this.service = new DataminerPoolManager();
}
@GET
@Path("/algorithm/stage")
@Produces("text/plain")
public String stageAlgorithm(
@QueryParam("algorithmPackageURL") String algorithmPackageURL,
@QueryParam("targetVRE") String targetVRE,
@QueryParam("category") String category,
public String stageAlgorithm(@QueryParam("algorithmPackageURL") String algorithmPackageURL,
@QueryParam("targetVRE") String targetVRE, @QueryParam("category") String category,
@QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException {
this.logger.debug("Stage algorithm method called");
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
//String env = context.application().getInitParameter("Environment");
return this.service.stageAlgorithm(algo,targetVRE,category,algorithm_type/*,env*/);
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
// String env = context.application().getInitParameter("Environment");
return this.service.stageAlgorithm(algo, targetVRE, category, algorithm_type/* ,env */);
}
@GET
@Path("/algorithm/add")
@Produces("text/plain")
public String publishAlgorithm(
@QueryParam("algorithmPackageURL") String algorithmPackageURL,
//@QueryParam("targetVREToken") String targetVREToken,
@QueryParam("targetVRE") String targetVRE,
@QueryParam("category") String category,
public String publishAlgorithm(@QueryParam("algorithmPackageURL") String algorithmPackageURL,
// @QueryParam("targetVREToken") String targetVREToken,
@QueryParam("targetVRE") String targetVRE, @QueryParam("category") String category,
@QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException {
this.logger.debug("Publish algorithm method called");
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
//String env = context.application().getInitParameter("Environment");
return this.service.publishAlgorithm(algo, /*targetVREToken,*/ targetVRE,category,algorithm_type/*,env*/);
// String env = context.application().getInitParameter("Environment");
return this.service.publishAlgorithm(algo, /* targetVREToken, */ targetVRE, category, algorithm_type/* ,env */);
}
/*
@ -87,7 +78,6 @@ public class RestPoolManager implements PoolManager
return service.getLogById(logUrl);
}
@GET
@Path("/monitor")
@Produces("text/plain")
@ -97,72 +87,77 @@ public class RestPoolManager implements PoolManager
this.logger.debug("Returning Log =" + logUrl);
return service.getMonitorById(logUrl);
}
@Override
@Override
public Algorithm extractAlgorithm(String url) throws IOException {
// TODO Auto-generated method stub
return null;
}
public static void main(String[] args) throws ObjectNotFound, Exception {
// System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
// // ProxySelector.setDefault(new
// // PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
//
// ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
// SecurityTokenProvider.instance.set("xxxx-xxx-xxx-xxx");
ScopeProvider.instance.set("/gcube/devNext");
SecurityTokenProvider.instance.set("xxx-xxx-xxx-xxx");
// AuthorizationEntry entry = authorizationService().get("xxx-xxx-xxx-xxx");
SecurityTokenProvider.instance.set("");
// AuthorizationEntry entry = authorizationService().get("xxxx-xxx-xxxxx-xxxx");
// System.out.println(entry.getContext());
RestPoolManager a = new RestPoolManager();
a.stageAlgorithm("http://data-d.d4science.org/TSt3cUpDTG1teUJMemxpcXplVXYzV1lBelVHTTdsYjlHbWJQNStIS0N6Yz0");
// //a.publishAlgorithm("http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "xxxx-xxx-xxx-xxx","/gcube/devNext/NextNext");
// //a.publishAlgorithm("http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "xxxx-xxx-xxxx-xxxx","/gcube/devNext/NextNext");
// // PoolManager aa = new DataminerPoolManager();
//
}
//Production Testing
// Production Testing
/*
stageAlgorithm(Rproto caller token,pacchetto, category)
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL
publishAlgorithm(Rproto caller token, pacchetto, category, target token, target prod vre)
node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/add?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL&targetVREToken=xxxx-xxx-xxx-xxx&targetVRE=/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab
getLogById(Rproto caller token, logid)
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxxx-xxxx-xxx-xxx&logUrl=
*/
//dev Testing
* stageAlgorithm(Rproto caller token,pacchetto, category)
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/algorithm/stage?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://
* data.d4science.org/
* dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=
* ICHTHYOP_MODEL
*
* publishAlgorithm(Rproto caller token, pacchetto, category, target token,
* target prod vre)
* node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/
* algorithm/add?gcube-token=xxxx-xxxx-xxx-xxx&algorithmPackageURL=http://data.
* d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&
* category=ICHTHYOP_MODEL&targetVREToken=xxxx-xxxx-xxxx&targetVRE=/d4science.
* research-infrastructures.eu/gCubeApps/RPrototypingLab
*
* getLogById(Rproto caller token, logid)
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/log?gcube-token=xxxx-xxx-xxx-xxx&logUrl=
*/
// dev Testing
/*
stageAlgorithm(dev_caller_vre_token,pacchetto, category)
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=xxx-xxxx-xxxx-xxx-xxxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL
publishAlgorithm(dev_caller_vre_token, pacchetto, category, target token, target prod vre)
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=450bb7f9-9e38-4bde-8f4d-f3296f95deba
getLogById(dev_caller_vre_token, logid)
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=426c8e35-a624-4710-b612-c90929c32c27 */
* stageAlgorithm(dev_caller_vre_token,pacchetto, category)
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/algorithm/stage?gcube-token=xxxx-xxxx-xxxx&algorithmPackageURL=http://
* data.d4science.org/
* dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=
* ICHTHYOP_MODEL
*
* publishAlgorithm(dev_caller_vre_token, pacchetto, category, target token,
* target prod vre)
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/log?gcube-token=xxxx-xxxx-xxx-xxxx&logUrl=450bb7f9-9e38-4bde-8f4d-
* f3296f95deba
*
* getLogById(dev_caller_vre_token, logid)
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=426c8e35-a624-4710-b612-
* c90929c32c27
*/
@Override
public void getLogId(Algorithm algo, String vre) {
@ -176,8 +171,6 @@ public class RestPoolManager implements PoolManager
return null;
}
@Override
public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException {
// TODO Auto-generated method stub
@ -196,37 +189,31 @@ public class RestPoolManager implements PoolManager
return null;
}
@Override
public List<String> updateSVN(String file, List<String> ldep) throws SVNException {
// TODO Auto-generated method stub
return null;
}
@Override
public String addAlgorithmToHost(Algorithm algo, String host, boolean test)
public String addAlgorithmToHost(Algorithm algo, String host, boolean test)
throws IOException, InterruptedException {
// TODO Auto-generated method stub
return null;
}
@Override
public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test)
throws IOException, InterruptedException {
public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test) throws IOException, InterruptedException {
// TODO Auto-generated method stub
return null;
}
@Override
public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException {
// TODO Auto-generated method stub
return null;
}
@Override
public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE)
throws IOException, InterruptedException {
@ -234,12 +221,4 @@ public class RestPoolManager implements PoolManager
return null;
}
}

7
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java
View File

@ -63,14 +63,13 @@ public class CheckMethodProduction extends CheckMethod{
//System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org",
// "xxxx-xxx-xxx-xxx"));
//System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
// "xxx-xxxx-xxx-xxx"));
// "xxx-xxxx-xxx-xx"));
try
{
a.checkMethod("dataminer-ghost-d.dev.d4science.org",
a.checkMethod("dataminer-ghost.cloud-dev.d4science.org",
"xxx-xxx-xxx-xxx");
} catch (Exception e)
{

7
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java
View File

@ -38,7 +38,7 @@ public class CheckMethodStaging extends CheckMethod{
//a.copyFromDmToSVN(aa);
// if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "xxxx-xxx-xxx-xxx")){
// if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "xxxx-xxxx-xxx-xxx")){
// System.out.println("AAA"); }
//
// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){
@ -63,7 +63,7 @@ public class CheckMethodStaging extends CheckMethod{
try
{
a.checkMethod("dataminer-ghost-d.dev.d4science.org",
"xxx-xxx-xxxx-xxx");
"xxxx-xxxx-xxxx-xxxx");
} catch (Exception e)
{
e.printStackTrace();
@ -77,7 +77,8 @@ public class CheckMethodStaging extends CheckMethod{
////
//ServiceConfiguration bp = new ServiceConfiguration();
////
//SecurityTokenProvider.instance.set("xxxx-xxx-xxx-xxx");
//SecurityTokenProvider.instance.set("xxx-xxx-xxx-xxx");
////
//if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); {
//System.out.println("ciao");