gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

Merge pull request 'Feature/24026' (!3) from Feature/24026 into master 

Reviewed-on: #3
This commit is contained in:
Giancarlo Panichi 2023-11-09 19:12:52 +01:00
parent 4a5eb67f73 033ce0b37b
commit e1d88418be
7 changed files with 86 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.md
View File

@ -1,7 +1,7 @@
# Changelog # Changelog
## [v2.7.2-SNAPSHOT] ## [v2.8.0-SNAPSHOT]
- Updated to gcube-smartgears-bom.2.5.0 - Updated to gcube-smartgears-bom.2.5.0

2
changelog.xml
View File

@ -1,5 +1,5 @@
<ReleaseNotes> <ReleaseNotes>
<Changeset component="org.gcube.dataanalysis.dataminer-pool-manager.2-7-2" date="2023-11-09"> <Changeset component="org.gcube.dataanalysis.dataminer-pool-manager.2-8-0" date="2023-11-09">
<Change>Updated to gcube-smartgears-bom.2.5.0</Change> <Change>Updated to gcube-smartgears-bom.2.5.0</Change>
</Changeset> </Changeset>
<Changeset component="org.gcube.dataanalysis.dataminer-pool-manager.2-7-1" date="2022-04-06"> <Changeset component="org.gcube.dataanalysis.dataminer-pool-manager.2-7-1" date="2022-04-06">

2
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataanalysis</groupId> <groupId>org.gcube.dataanalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId> <artifactId>dataminer-pool-manager</artifactId>
<packaging>war</packaging> <packaging>war</packaging>
<version>2.7.2-SNAPSHOT</version> <version>2.8.0-SNAPSHOT</version>
<name>dataminer-pool-manager</name> <name>dataminer-pool-manager</name>
<description>DataMiner Pool Manager is a service to support the integration of algorithms in D4Science Infrastructure</description> <description>DataMiner Pool Manager is a service to support the integration of algorithms in D4Science Infrastructure</description>

9
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/HAProxy.java
View File

@ -20,6 +20,9 @@ import au.com.bytecode.opencsv.CSVReader;
public class HAProxy { public class HAProxy {
private static final String token1 = "";
private static final String token2 = "";
private Logger logger; private Logger logger;
private CSVReader reader; private CSVReader reader;
@ -142,16 +145,16 @@ public class HAProxy {
HAProxy a = new HAProxy(); HAProxy a = new HAProxy();
//ScopeProvider.instance.set("/gcube/devNext/NextNext"); //ScopeProvider.instance.set("/gcube/devNext/NextNext");
//ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); //ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SecurityTokenProvider.instance.set(""); SecurityTokenProvider.instance.set(token1);
CheckPermission test = new CheckPermission(); CheckPermission test = new CheckPermission();
CheckPermission.apply("", "/gcube/devNext/NextNext"); CheckPermission.apply(token2, "/gcube/devNext/NextNext");
//ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); //ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
// System.out.println(a.getHProxy()); // System.out.println(a.getHProxy());
// System.out.println(a.MapCluster()); // System.out.println(a.MapCluster());
//System.out.println(a.listDataMinersByCluster("","/gcube/devNext/NextNext")); //System.out.println(a.listDataMinersByCluster("xxx-xxx-xxx-xxx","/gcube/devNext/NextNext"));
// System.out.println(a.listDataMinersByCluster()); // System.out.println(a.listDataMinersByCluster());
// List<Dependency> list = new LinkedList<Dependency>(); // List<Dependency> list = new LinkedList<Dependency>();

161
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -1,6 +1,5 @@
package org.gcube.dataanalysis.dataminer.poolmanager.rest; package org.gcube.dataanalysis.dataminer.poolmanager.rest;
import java.io.IOException; import java.io.IOException;
import java.net.MalformedURLException; import java.net.MalformedURLException;
import java.net.URL; import java.net.URL;
@ -24,52 +23,44 @@ import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
@Path("/") @Path("/")
public class RestPoolManager implements PoolManager public class RestPoolManager implements PoolManager {
{ // @Context ServletContext context;
//@Context ServletContext context;
private final Logger logger; private final Logger logger;
private DataminerPoolManager service; private DataminerPoolManager service;
//@Context // @Context
//private ApplicationContext context = ContextProvider.get(); // private ApplicationContext context = ContextProvider.get();
public RestPoolManager() public RestPoolManager() {
{
this.logger = LoggerFactory.getLogger(RestPoolManager.class); this.logger = LoggerFactory.getLogger(RestPoolManager.class);
this.service = new DataminerPoolManager(); this.service = new DataminerPoolManager();
} }
@GET @GET
@Path("/algorithm/stage") @Path("/algorithm/stage")
@Produces("text/plain") @Produces("text/plain")
public String stageAlgorithm( public String stageAlgorithm(@QueryParam("algorithmPackageURL") String algorithmPackageURL,
@QueryParam("algorithmPackageURL") String algorithmPackageURL, @QueryParam("targetVRE") String targetVRE, @QueryParam("category") String category,
@QueryParam("targetVRE") String targetVRE,
@QueryParam("category") String category,
@QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException { @QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException {
this.logger.debug("Stage algorithm method called"); this.logger.debug("Stage algorithm method called");
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
//String env = context.application().getInitParameter("Environment"); // String env = context.application().getInitParameter("Environment");
return this.service.stageAlgorithm(algo,targetVRE,category,algorithm_type/*,env*/); return this.service.stageAlgorithm(algo, targetVRE, category, algorithm_type/* ,env */);
} }
@GET @GET
@Path("/algorithm/add") @Path("/algorithm/add")
@Produces("text/plain") @Produces("text/plain")
public String publishAlgorithm( public String publishAlgorithm(@QueryParam("algorithmPackageURL") String algorithmPackageURL,
@QueryParam("algorithmPackageURL") String algorithmPackageURL, // @QueryParam("targetVREToken") String targetVREToken,
//@QueryParam("targetVREToken") String targetVREToken, @QueryParam("targetVRE") String targetVRE, @QueryParam("category") String category,
@QueryParam("targetVRE") String targetVRE,
@QueryParam("category") String category,
@QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException { @QueryParam("algorithm_type") String algorithm_type) throws IOException, InterruptedException {
this.logger.debug("Publish algorithm method called"); this.logger.debug("Publish algorithm method called");
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL); Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
//String env = context.application().getInitParameter("Environment"); // String env = context.application().getInitParameter("Environment");
return this.service.publishAlgorithm(algo, /*targetVREToken,*/ targetVRE,category,algorithm_type/*,env*/); return this.service.publishAlgorithm(algo, /* targetVREToken, */ targetVRE, category, algorithm_type/* ,env */);
} }
/* /*
@ -87,7 +78,6 @@ public class RestPoolManager implements PoolManager
return service.getLogById(logUrl); return service.getLogById(logUrl);
} }
@GET @GET
@Path("/monitor") @Path("/monitor")
@Produces("text/plain") @Produces("text/plain")
@ -97,72 +87,77 @@ public class RestPoolManager implements PoolManager
this.logger.debug("Returning Log =" + logUrl); this.logger.debug("Returning Log =" + logUrl);
return service.getMonitorById(logUrl); return service.getMonitorById(logUrl);
} }
@Override
@Override
public Algorithm extractAlgorithm(String url) throws IOException { public Algorithm extractAlgorithm(String url) throws IOException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
public static void main(String[] args) throws ObjectNotFound, Exception { public static void main(String[] args) throws ObjectNotFound, Exception {
// System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); // System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
// // ProxySelector.setDefault(new // // ProxySelector.setDefault(new
// // PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); // // PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
// //
// ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
// SecurityTokenProvider.instance.set("xxxx-xxx-xxx-xxx");
ScopeProvider.instance.set("/gcube/devNext"); ScopeProvider.instance.set("/gcube/devNext");
SecurityTokenProvider.instance.set("xxx-xxx-xxx-xxx"); SecurityTokenProvider.instance.set("");
// AuthorizationEntry entry = authorizationService().get("xxxx-xxx-xxxxx-xxxx");
// AuthorizationEntry entry = authorizationService().get("xxx-xxx-xxx-xxx");
// System.out.println(entry.getContext()); // System.out.println(entry.getContext());
RestPoolManager a = new RestPoolManager(); RestPoolManager a = new RestPoolManager();
a.stageAlgorithm("http://data-d.d4science.org/TSt3cUpDTG1teUJMemxpcXplVXYzV1lBelVHTTdsYjlHbWJQNStIS0N6Yz0"); a.stageAlgorithm("http://data-d.d4science.org/TSt3cUpDTG1teUJMemxpcXplVXYzV1lBelVHTTdsYjlHbWJQNStIS0N6Yz0");
// //a.publishAlgorithm("http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "xxxx-xxx-xxx-xxx","/gcube/devNext/NextNext"); // //a.publishAlgorithm("http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "xxxx-xxx-xxxx-xxxx","/gcube/devNext/NextNext");
// // PoolManager aa = new DataminerPoolManager(); // // PoolManager aa = new DataminerPoolManager();
// //
} }
// Production Testing
//Production Testing
/* /*
stageAlgorithm(Rproto caller token,pacchetto, category) * stageAlgorithm(Rproto caller token,pacchetto, category)
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL * http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/algorithm/stage?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://
publishAlgorithm(Rproto caller token, pacchetto, category, target token, target prod vre) * data.d4science.org/
node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/add?gcube-token=xxx-xxx-xxx-xxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL&targetVREToken=xxxx-xxx-xxx-xxx&targetVRE=/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab * dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=
* ICHTHYOP_MODEL
getLogById(Rproto caller token, logid) *
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxxx-xxxx-xxx-xxx&logUrl= * publishAlgorithm(Rproto caller token, pacchetto, category, target token,
*/ * target prod vre)
* node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/
* algorithm/add?gcube-token=xxxx-xxxx-xxx-xxx&algorithmPackageURL=http://data.
//dev Testing * d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&
* category=ICHTHYOP_MODEL&targetVREToken=xxxx-xxxx-xxxx&targetVRE=/d4science.
* research-infrastructures.eu/gCubeApps/RPrototypingLab
*
* getLogById(Rproto caller token, logid)
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/log?gcube-token=xxxx-xxx-xxx-xxx&logUrl=
*/
// dev Testing
/* /*
stageAlgorithm(dev_caller_vre_token,pacchetto, category) * stageAlgorithm(dev_caller_vre_token,pacchetto, category)
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=xxx-xxxx-xxxx-xxx-xxxx&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL * http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/algorithm/stage?gcube-token=xxxx-xxxx-xxxx&algorithmPackageURL=http://
publishAlgorithm(dev_caller_vre_token, pacchetto, category, target token, target prod vre) * data.d4science.org/
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=450bb7f9-9e38-4bde-8f4d-f3296f95deba * dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=
* ICHTHYOP_MODEL
getLogById(dev_caller_vre_token, logid) *
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=426c8e35-a624-4710-b612-c90929c32c27 */ * publishAlgorithm(dev_caller_vre_token, pacchetto, category, target token,
* target prod vre)
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/log?gcube-token=xxxx-xxxx-xxx-xxxx&logUrl=450bb7f9-9e38-4bde-8f4d-
* f3296f95deba
*
* getLogById(dev_caller_vre_token, logid)
* http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/
* rest/log?gcube-token=xxx-xxx-xxx-xxx&logUrl=426c8e35-a624-4710-b612-
* c90929c32c27
*/
@Override @Override
public void getLogId(Algorithm algo, String vre) { public void getLogId(Algorithm algo, String vre) {
@ -176,8 +171,6 @@ public class RestPoolManager implements PoolManager
return null; return null;
} }
@Override @Override
public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
@ -196,37 +189,31 @@ public class RestPoolManager implements PoolManager
return null; return null;
} }
@Override @Override
public List<String> updateSVN(String file, List<String> ldep) throws SVNException { public List<String> updateSVN(String file, List<String> ldep) throws SVNException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
@Override @Override
public String addAlgorithmToHost(Algorithm algo, String host, boolean test) public String addAlgorithmToHost(Algorithm algo, String host, boolean test)
throws IOException, InterruptedException { throws IOException, InterruptedException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
@Override @Override
public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test) public String addAlgorithmToVRE(Algorithm algo, String vre, boolean test) throws IOException, InterruptedException {
throws IOException, InterruptedException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
@Override @Override
public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException { public String stageAlgorithm(String algorithmPackageURL) throws IOException, InterruptedException {
// TODO Auto-generated method stub // TODO Auto-generated method stub
return null; return null;
} }
@Override @Override
public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE) public String publishAlgorithm(String algorithmPackageURL, String targetVREToken, String targetVRE)
throws IOException, InterruptedException { throws IOException, InterruptedException {
@ -234,12 +221,4 @@ public class RestPoolManager implements PoolManager
return null; return null;
} }
} }

7
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java
View File

@ -63,14 +63,13 @@ public class CheckMethodProduction extends CheckMethod{
//System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org", //System.out.println(a.checkMethod("dataminer-ghost-t.pre.d4science.org",
// "xxxx-xxx-xxx-xxx"));
//System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", // "xxx-xxxx-xxx-xx"));
// "xxx-xxxx-xxx-xxx"));
try try
{ {
a.checkMethod("dataminer-ghost-d.dev.d4science.org", a.checkMethod("dataminer-ghost.cloud-dev.d4science.org",
"xxx-xxx-xxx-xxx"); "xxx-xxx-xxx-xxx");
} catch (Exception e) } catch (Exception e)
{ {

7
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java
View File

@ -38,7 +38,7 @@ public class CheckMethodStaging extends CheckMethod{
//a.copyFromDmToSVN(aa); //a.copyFromDmToSVN(aa);
// if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "xxxx-xxx-xxx-xxx")){ // if (a.checkMethod("dataminer-ghost-d.dev.d4science.org", "xxxx-xxxx-xxx-xxx")){
// System.out.println("AAA"); } // System.out.println("AAA"); }
// //
// if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){ // if (a.doesExist("/home/gcube/wps_algorithms/algorithms/WINDOWS_BLACK_BOX_EXAMPLE.jar")){
@ -63,7 +63,7 @@ public class CheckMethodStaging extends CheckMethod{
try try
{ {
a.checkMethod("dataminer-ghost-d.dev.d4science.org", a.checkMethod("dataminer-ghost-d.dev.d4science.org",
"xxx-xxx-xxxx-xxx"); "xxxx-xxxx-xxxx-xxxx");
} catch (Exception e) } catch (Exception e)
{ {
e.printStackTrace(); e.printStackTrace();
@ -77,7 +77,8 @@ public class CheckMethodStaging extends CheckMethod{
//// ////
//ServiceConfiguration bp = new ServiceConfiguration(); //ServiceConfiguration bp = new ServiceConfiguration();
//// ////
//SecurityTokenProvider.instance.set("xxxx-xxx-xxx-xxx"); //SecurityTokenProvider.instance.set("xxx-xxx-xxx-xxx");
//// ////
//if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); { //if (a.checkMethod(bp.getStagingHost(), SecurityTokenProvider.instance.get())&&a.algoExists(aa)); {
//System.out.println("ciao"); //System.out.println("ciao");