From 477932e99a522ef4ccd376cd76ba3d1820471740 Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Tue, 4 Apr 2017 17:32:53 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146567 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 12 ++- .../poolmanager/clients/ISClient.java | 75 ++++++++++++++++++- .../service/DataminerPoolManager.java | 1 + 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 4367204..fb6f340 100644 --- a/pom.xml +++ b/pom.xml @@ -24,10 +24,10 @@ - + org.gcube.core common-smartgears - + provided @@ -45,6 +45,12 @@ commons-configuration 1.10 + + + net.sf.opencsv + opencsv + 2.3 + org.gcube.resources.discovery ic-client @@ -77,7 +83,7 @@ slf4j-api provided - + diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index 57b6a1b..2101717 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -69,19 +69,36 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients; import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; -import java.util.Collection; +import java.io.BufferedReader; ***REMOVED*** +import java.io.InputStreamReader; +***REMOVED*** +***REMOVED*** +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedList; +***REMOVED*** +import java.util.Map; +***REMOVED*** +import java.util.TreeMap; +import java.util.TreeSet; import java.util.Vector; import org.gcube.common.resources.gcore.ServiceEndpoint; ***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import au.com.bytecode.opencsv.CSVReader; + public class ISClient ***REMOVED*** + private CSVReader reader; + public Host getDataminer(String hostname) ***REMOVED*** Host h = new Host(); boolean remote2 = true; @@ -101,6 +118,55 @@ public class ISClient ***REMOVED*** ***REMOVED*** + +***REMOVED*** return the HProxy hostname in the VRE +public String getHProxy()***REMOVED*** + Host h = new Host(); + SimpleQuery query = queryFor(ServiceEndpoint.class); + query.addCondition("$resource/Profile/Platform/Name/text() eq 'DataMinerAlone'"); + DiscoveryClient client = clientFor(ServiceEndpoint.class); + List resources = client.submit(query); + h.setName(resources.get(0).profile().runtime().hostedOn()); + return h.getName(); +***REMOVED*** + + +public Cluster getClusterByHProxy() throws IOException ***REMOVED*** + Cluster cl = new Cluster(); + String HProxy = this.getHProxy(); + URL stockURL = new URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); + BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream())); + reader = new CSVReader(in); + String [] nextLine; + while ((nextLine = reader.readNext()) != null) ***REMOVED*** + ***REMOVED***rules to add + if (HProxy.contains(nextLine[0]))***REMOVED*** + cl.setName(nextLine[0]); + ***REMOVED*** + ***REMOVED*** + return cl; + +***REMOVED*** + + + public List listDataMinersByCluster() throws IOException ***REMOVED*** + Cluster cluster = this.getClusterByHProxy(); + List out = new LinkedList(); + URL stockURL = new URL("http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0"); + BufferedReader in = new BufferedReader(new InputStreamReader(stockURL.openStream())); + reader = new CSVReader(in, ','); + String[] nextLine; + while ((nextLine = reader.readNext()) != null) ***REMOVED*** + if (nextLine[0].equals(cluster.getName())) ***REMOVED*** + Host a = new Host(); + a.setName(nextLine[1]); + out.add(a); + System.out.println(a.getFullyQualifiedName()); + ***REMOVED*** + ***REMOVED*** + return out; + +***REMOVED*** /** * Return the list of hosts (dataminers) in a given VRE @@ -144,10 +210,13 @@ public class ISClient ***REMOVED*** ***REMOVED*** ***REMOVED*** - public static void main(String[] args) ***REMOVED*** + public static void main(String[] args) throws IOException ***REMOVED*** ISClient a = new ISClient(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); - System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain()); + ***REMOVED***System.out.println(a.getHProxy()); + System.out.println(a.getClusterByHProxy()); + System.out.println(a.listDataMinersByCluster()); + ***REMOVED***System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain()); ***REMOVED***System.out.println(a.listDataminersInVRE()); ***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index a687c6f..65532b2 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -523,6 +523,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED*** ***REMOVED*** create the cluster (dataminers in the vre) Cluster cluster = new Cluster(); + ***REMOVED***for (Host h : new ISClient().listDataMinersByCluster()) ***REMOVED*** for (Host h : new ISClient().listDataminersInVRE()) ***REMOVED*** cluster.addHost(h); ***REMOVED***