gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146567 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-04-04 17:32:53 +00:00
parent 7958aeecfa
commit e99eccffb3
4 changed files with 94 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@ -24,10 +24,10 @@
</properties>
<dependencies>
<dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version>-->
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version> -->
<scope>provided</scope>
</dependency>
<dependency>
@ -45,6 +45,12 @@
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
<artifactId>ic-client</artifactId>
@ -77,7 +83,7 @@
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>

73
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -69,19 +69,36 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.Vector;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import au.com.bytecode.opencsv.CSVReader;
public class ISClient {
private CSVReader reader;
public Host getDataminer(String hostname) {
Host h = new Host();
boolean remote2 = true;
@ -101,6 +118,55 @@ public class ISClient {
}
// return the HProxy hostname in the VRE
public String getHProxy(){
Host h = new Host();
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMinerAlone'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> resources = client.submit(query);
h.setName(resources.get(0).profile().runtime().hostedOn());
return h.getName();
}
public Cluster getClusterByHProxy() throws IOException {
Cluster cl = new Cluster();
String HProxy = this.getHProxy();
URL stockURL = new URL("http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0");
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
reader = new CSVReader(in);
String [] nextLine;
while ((nextLine = reader.readNext()) != null) {
//rules to add
if (HProxy.contains(nextLine[0])){
cl.setName(nextLine[0]);
}
}
return cl;
}
public List<Host> listDataMinersByCluster() throws IOException {
Cluster cluster = this.getClusterByHProxy();
List<Host> out = new LinkedList<Host>();
URL stockURL = new URL("http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0");
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
reader = new CSVReader(in, ',');
String[] nextLine;
while ((nextLine = reader.readNext()) != null) {
if (nextLine[0].equals(cluster.getName())) {
Host a = new Host();
a.setName(nextLine[1]);
out.add(a);
System.out.println(a.getFullyQualifiedName());
}
}
return out;
}
/**
* Return the list of hosts (dataminers) in a given VRE
@ -144,10 +210,13 @@ public class ISClient {
}
}
public static void main(String[] args) {
public static void main(String[] args) throws IOException {
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
//System.out.println(a.getHProxy());
System.out.println(a.getClusterByHProxy());
System.out.println(a.listDataMinersByCluster());
//System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
//System.out.println(a.listDataminersInVRE());
}
}

62
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -151,6 +151,7 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.Produces;
@ -188,10 +189,14 @@ public class RestPoolManager implements PoolManager {
@QueryParam("name") String name,
@QueryParam("description") String description,
@QueryParam("category") String category,
@QueryParam("algorithmType") String algorithmType,
@QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
@DefaultValue("N") @QueryParam("skipJava") String skipJava,
@DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException {
Algorithm algo= this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava);
//service.addAlgToIs(algo);
if (publish){
service.addAlgToIs(algo);
}
return service.addAlgorithmToVRE(algo, vre);
}
@ -204,10 +209,9 @@ public class RestPoolManager implements PoolManager {
@QueryParam("name") String name,
@QueryParam("description") String description,
@QueryParam("category") String category,
@QueryParam("algorithmType") String algorithmType,
@QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
@DefaultValue("N") @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
Algorithm algo= this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, skipJava);
//service.addAlgToIs(algo);
return service.addAlgorithmToHost(algo, hostname);
}
@ -221,11 +225,9 @@ public class RestPoolManager implements PoolManager {
String category,
String algorithmType,
String skipJava) throws IOException, InterruptedException {
// TODO Auto-generated method stub
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
Algorithm algo = service.extractAlgorithm(algorithm);
// UUID uuid = UUID.randomUUID();
// algo.setId(name+"@"+uuid);
if (algo.getCategory() == null){
algo.setCategory(category);
@ -250,45 +252,7 @@ public class RestPoolManager implements PoolManager {
return algo;
}
// @GET
// @Path("/add")
// @Produces("text/plain")
// public String addAlgorithmToVRE(
// @QueryParam("algorithm") String algorithm,
// @QueryParam("vre") String vre,
// @QueryParam("name") String name,
// @QueryParam("description") String description,
// @QueryParam("category") String category,
// @QueryParam("algorithmType") String algorithmType,
// @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
// // TODO Auto-generated method stub
// LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
// Algorithm algo = service.extractAlgorithm(algorithm);
//
// if (algo.getCategory() == null){
// algo.setCategory(category);
// } else category = algo.getCategory();
//
// if (algo.getAlgorithmType() == null){
// algo.setAlgorithmType(algorithmType);
// } else algorithmType = algo.getCategory();
//
// if (algo.getSkipJava() == null){
// algo.setSkipJava(skipJava);
// } else skipJava = algo.getSkipJava();
//
// if (algo.getName() == null){
// algo.setCategory(name);
// } else name = algo.getName();
//
// if (algo.getDescription() == null){
// algo.setDescription(description);;
// } else description = algo.getDescription();
//
// return service.addAlgorithmToVRE(algo, vre);
// }
@GET
@Path("/log")
@ -328,7 +292,7 @@ public class RestPoolManager implements PoolManager {
//null, null, "test", "transducerers", "N");
a.addAlgorithmToHost(
"http://data.d4science.org/YkErc2drUFRZUlV1ZkdqVnlhVW1mNjYvZ1ZiSHRGTUFHbWJQNStIS0N6Yz0",
"http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
"dataminer1-devnext.d4science.org",
"ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N");
}

1
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -523,6 +523,7 @@ public class DataminerPoolManager implements PoolManager {
// create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
//for (Host h : new ISClient().listDataMinersByCluster()) {
for (Host h : new ISClient().listDataminersInVRE()) {
cluster.addHost(h);
}