gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146567 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-04-04 17:32:53 +00:00
parent 5144646ca3
commit 477932e99a
3 changed files with 82 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pom.xml
View File

@ -24,10 +24,10 @@
</properties>
<dependencies>
<dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version>-->
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version> -->
<scope>provided</scope>
</dependency>
<dependency>
@ -45,6 +45,12 @@
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
<artifactId>ic-client</artifactId>
@ -77,7 +83,7 @@
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>

75
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -69,19 +69,36 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients;
import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.Collection;
import java.io.BufferedReader;
***REMOVED***
import java.io.InputStreamReader;
***REMOVED***
***REMOVED***
import java.util.Collection;
import java.util.LinkedHashMap;
import java.util.LinkedList;
***REMOVED***
import java.util.Map;
***REMOVED***
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.Vector;
import org.gcube.common.resources.gcore.ServiceEndpoint;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import au.com.bytecode.opencsv.CSVReader;
public class ISClient ***REMOVED***
private CSVReader reader;
public Host getDataminer(String hostname) ***REMOVED***
Host h = new Host();
boolean remote2 = true;
@ -101,6 +118,55 @@ public class ISClient ***REMOVED***
***REMOVED***
***REMOVED*** return the HProxy hostname in the VRE
public String getHProxy()***REMOVED***
Host h = new Host();
SimpleQuery query = queryFor(ServiceEndpoint.class);
query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMinerAlone'");
DiscoveryClient<ServiceEndpoint> client = clientFor(ServiceEndpoint.class);
List<ServiceEndpoint> resources = client.submit(query);
h.setName(resources.get(0).profile().runtime().hostedOn());
return h.getName();
***REMOVED***
public Cluster getClusterByHProxy() throws IOException ***REMOVED***
Cluster cl = new Cluster();
String HProxy = this.getHProxy();
URL stockURL = new URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0");
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
reader = new CSVReader(in);
String [] nextLine;
while ((nextLine = reader.readNext()) != null) ***REMOVED***
***REMOVED***rules to add
if (HProxy.contains(nextLine[0]))***REMOVED***
cl.setName(nextLine[0]);
***REMOVED***
***REMOVED***
return cl;
***REMOVED***
public List<Host> listDataMinersByCluster() throws IOException ***REMOVED***
Cluster cluster = this.getClusterByHProxy();
List<Host> out = new LinkedList<Host>();
URL stockURL = new URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0");
BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream()));
reader = new CSVReader(in, ',');
String[] nextLine;
while ((nextLine = reader.readNext()) != null) ***REMOVED***
if (nextLine[0].equals(cluster.getName())) ***REMOVED***
Host a = new Host();
a.setName(nextLine[1]);
out.add(a);
System.out.println(a.getFullyQualifiedName());
***REMOVED***
***REMOVED***
return out;
***REMOVED***
/**
* Return the list of hosts (dataminers) in a given VRE
@ -144,10 +210,13 @@ public class ISClient ***REMOVED***
***REMOVED***
***REMOVED***
public static void main(String[] args) ***REMOVED***
public static void main(String[] args) throws IOException ***REMOVED***
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
***REMOVED***System.out.println(a.getHProxy());
System.out.println(a.getClusterByHProxy());
System.out.println(a.listDataMinersByCluster());
***REMOVED***System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
***REMOVED***System.out.println(a.listDataminersInVRE());
***REMOVED***
***REMOVED***

1
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -523,6 +523,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster();
***REMOVED***for (Host h : new ISClient().listDataMinersByCluster()) ***REMOVED***
for (Host h : new ISClient().listDataminersInVRE()) ***REMOVED***
cluster.addHost(h);
***REMOVED***