git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146567 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
64a7d37a20
commit
e57b57e59c
12
pom.xml
12
pom.xml
|
|
@ -24,10 +24,10 @@
|
|||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.core</groupId>
|
||||
<artifactId>common-smartgears</artifactId>
|
||||
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version>-->
|
||||
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version> -->
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
|
|
@ -45,6 +45,12 @@
|
|||
<artifactId>commons-configuration</artifactId>
|
||||
<version>1.10</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>net.sf.opencsv</groupId>
|
||||
<artifactId>opencsv</artifactId>
|
||||
<version>2.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.resources.discovery</groupId>
|
||||
<artifactId>ic-client</artifactId>
|
||||
|
|
@ -77,7 +83,7 @@
|
|||
<artifactId>slf4j-api</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
|
|
|||
|
|
@ -69,19 +69,36 @@
|
|||
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
||||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||
|
||||
import java.util.Collection;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import java.net.MalformedURLException;
|
||||
***REMOVED***
|
||||
import java.util.Collection;
|
||||
import java.util.LinkedHashMap;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.TreeSet;
|
||||
import java.util.Vector;
|
||||
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
|
||||
***REMOVED***
|
||||
|
||||
public class ISClient {
|
||||
|
||||
***REMOVED***
|
||||
|
||||
public Host getDataminer(String hostname) {
|
||||
Host h = new Host();
|
||||
boolean remote2 = true;
|
||||
|
|
@ -101,6 +118,55 @@ public class ISClient {
|
|||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED*** return the HProxy hostname in the VRE
|
||||
public String getHProxy(){
|
||||
Host h = new Host();
|
||||
SimpleQuery query = queryFor(ServiceEndpoint.class);
|
||||
query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMinerAlone'");
|
||||
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
|
||||
List<ServiceEndpoint> resources = client.submit(query);
|
||||
h.setName(resources.get(0).profile().runtime().hostedOn());
|
||||
return h.getName();
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public Cluster getClusterByHProxy() throws IOException {
|
||||
Cluster cl = new Cluster();
|
||||
String HProxy = this.getHProxy();
|
||||
URL stockURL = new URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0");
|
||||
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
|
||||
reader = new CSVReader(in);
|
||||
String [] nextLine;
|
||||
while ((nextLine = reader.readNext()) != null) {
|
||||
***REMOVED***rules to add
|
||||
if (HProxy.contains(nextLine[0])){
|
||||
cl.setName(nextLine[0]);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return cl;
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public List<Host> listDataMinersByCluster() throws IOException {
|
||||
Cluster cluster = this.getClusterByHProxy();
|
||||
***REMOVED***
|
||||
URL stockURL = new URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0");
|
||||
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
|
||||
reader = new CSVReader(in, ',');
|
||||
String[] nextLine;
|
||||
while ((nextLine = reader.readNext()) != null) {
|
||||
if (nextLine[0].equals(cluster.getName())) {
|
||||
***REMOVED***
|
||||
a.setName(nextLine[1]);
|
||||
***REMOVED***
|
||||
System.out.println(a.getFullyQualifiedName());
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
||||
/**
|
||||
* Return the list of hosts (dataminers) in a given VRE
|
||||
|
|
@ -144,10 +210,13 @@ public class ISClient {
|
|||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public static void main(String[] args) {
|
||||
public static void main(String[] args) throws IOException {
|
||||
ISClient a = new ISClient();
|
||||
ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
||||
System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
|
||||
***REMOVED***System.out.println(a.getHProxy());
|
||||
System.out.println(a.getClusterByHProxy());
|
||||
System.out.println(a.listDataMinersByCluster());
|
||||
***REMOVED***System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
|
||||
***REMOVED***System.out.println(a.listDataminersInVRE());
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -151,6 +151,7 @@ import java.util.Random;
|
|||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import javax.ws.rs.DefaultValue;
|
||||
import javax.ws.rs.GET;
|
||||
import javax.ws.rs.Path;
|
||||
import javax.ws.rs.Produces;
|
||||
|
|
@ -188,10 +189,14 @@ public class RestPoolManager implements PoolManager {
|
|||
@QueryParam("name") String name,
|
||||
@QueryParam("description") String description,
|
||||
@QueryParam("category") String category,
|
||||
@QueryParam("algorithmType") String algorithmType,
|
||||
@QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
|
||||
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
|
||||
@DefaultValue("N") @QueryParam("skipJava") String skipJava,
|
||||
@DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException {
|
||||
Algorithm algo= this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava);
|
||||
***REMOVED***service.addAlgToIs(algo);
|
||||
|
||||
if (publish){
|
||||
service.addAlgToIs(algo);
|
||||
***REMOVED***
|
||||
return service.addAlgorithmToVRE(algo, vre);
|
||||
***REMOVED***
|
||||
|
||||
|
|
@ -204,10 +209,9 @@ public class RestPoolManager implements PoolManager {
|
|||
@QueryParam("name") String name,
|
||||
@QueryParam("description") String description,
|
||||
@QueryParam("category") String category,
|
||||
@QueryParam("algorithmType") String algorithmType,
|
||||
@QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
|
||||
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
|
||||
@DefaultValue("N") @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
|
||||
Algorithm algo= this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, skipJava);
|
||||
***REMOVED***service.addAlgToIs(algo);
|
||||
return service.addAlgorithmToHost(algo, hostname);
|
||||
***REMOVED***
|
||||
|
||||
|
|
@ -221,11 +225,9 @@ public class RestPoolManager implements PoolManager {
|
|||
String category,
|
||||
String algorithmType,
|
||||
String skipJava) throws IOException, InterruptedException {
|
||||
***REMOVED*** TODO Auto-generated method stub
|
||||
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
|
||||
Algorithm algo = service.extractAlgorithm(algorithm);
|
||||
***REMOVED*** UUID uuid = UUID.randomUUID();
|
||||
***REMOVED*** algo.setId(name+"@"+uuid);
|
||||
|
||||
|
||||
if (algo.getCategory() == null){
|
||||
algo.setCategory(category);
|
||||
|
|
@ -250,45 +252,7 @@ public class RestPoolManager implements PoolManager {
|
|||
return algo;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
***REMOVED*** @GET
|
||||
***REMOVED*** @Path("/add")
|
||||
***REMOVED*** @Produces("text/plain")
|
||||
***REMOVED*** public String addAlgorithmToVRE(
|
||||
***REMOVED*** @QueryParam("algorithm") String algorithm,
|
||||
***REMOVED*** @QueryParam("vre") String vre,
|
||||
***REMOVED*** @QueryParam("name") String name,
|
||||
***REMOVED*** @QueryParam("description") String description,
|
||||
***REMOVED*** @QueryParam("category") String category,
|
||||
***REMOVED*** @QueryParam("algorithmType") String algorithmType,
|
||||
***REMOVED*** @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
|
||||
***REMOVED*** ***REMOVED*** TODO Auto-generated method stub
|
||||
***REMOVED*** LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
|
||||
***REMOVED*** Algorithm algo = service.extractAlgorithm(algorithm);
|
||||
***REMOVED***
|
||||
***REMOVED*** if (algo.getCategory() == null){
|
||||
***REMOVED*** algo.setCategory(category);
|
||||
***REMOVED*** ***REMOVED*** else category = algo.getCategory();
|
||||
***REMOVED***
|
||||
***REMOVED*** if (algo.getAlgorithmType() == null){
|
||||
***REMOVED*** algo.setAlgorithmType(algorithmType);
|
||||
***REMOVED*** ***REMOVED*** else algorithmType = algo.getCategory();
|
||||
***REMOVED***
|
||||
***REMOVED*** if (algo.getSkipJava() == null){
|
||||
***REMOVED*** algo.setSkipJava(skipJava);
|
||||
***REMOVED*** ***REMOVED*** else skipJava = algo.getSkipJava();
|
||||
***REMOVED***
|
||||
***REMOVED*** if (algo.getName() == null){
|
||||
***REMOVED*** algo.setCategory(name);
|
||||
***REMOVED*** ***REMOVED*** else name = algo.getName();
|
||||
***REMOVED***
|
||||
***REMOVED*** if (algo.getDescription() == null){
|
||||
***REMOVED*** algo.setDescription(description);;
|
||||
***REMOVED*** ***REMOVED*** else description = algo.getDescription();
|
||||
***REMOVED***
|
||||
***REMOVED*** return service.addAlgorithmToVRE(algo, vre);
|
||||
***REMOVED******REMOVED***
|
||||
|
||||
|
||||
|
||||
@GET
|
||||
@Path("/log")
|
||||
|
|
@ -328,7 +292,7 @@ public class RestPoolManager implements PoolManager {
|
|||
***REMOVED***null, null, "test", "transducerers", "N");
|
||||
|
||||
a.addAlgorithmToHost(
|
||||
"http:***REMOVED***data.d4science.org/YkErc2drUFRZUlV1ZkdqVnlhVW1mNjYvZ1ZiSHRGTUFHbWJQNStIS0N6Yz0",
|
||||
"http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
|
||||
"dataminer1-devnext.d4science.org",
|
||||
"ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N");
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -523,6 +523,7 @@ public class DataminerPoolManager implements PoolManager {
|
|||
|
||||
***REMOVED*** create the cluster (dataminers in the vre)
|
||||
Cluster cluster = new Cluster();
|
||||
***REMOVED***for (Host h : new ISClient().listDataMinersByCluster()) {
|
||||
for (Host h : new ISClient().listDataminersInVRE()) {
|
||||
cluster.addHost(h);
|
||||
***REMOVED***
|
||||
|
|
|
|||
Loading…
Reference in New Issue