git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148467 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
19bb79eae9
commit
3f523d3862
|
|
@ -10,6 +10,7 @@ import java.io.File;
|
|||
import java.io.InputStream;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.PrintStream;
|
||||
import java.io.StringWriter;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import java.util.ArrayList;
|
||||
|
|
@ -27,15 +28,26 @@ import java.util.TreeMap;
|
|||
import java.util.TreeSet;
|
||||
import java.util.Vector;
|
||||
|
||||
***REMOVED***
|
||||
import org.gcube.common.resources.gcore.GenericResource;
|
||||
import org.gcube.common.resources.gcore.Resources;
|
||||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
|
||||
***REMOVED***
|
||||
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
|
||||
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
||||
import org.gcube.informationsystem.publisher.ScopedPublisher;
|
||||
import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.tmatesoft.svn.core.SVNDepth;
|
||||
***REMOVED***
|
||||
import org.tmatesoft.svn.core.SVNURL;
|
||||
|
|
@ -52,6 +64,7 @@ import org.tmatesoft.svn.core.wc.SVNWCUtil;
|
|||
import au.com.bytecode.opencsv.CSVReader;
|
||||
|
||||
public class ISClient ***REMOVED***
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(ISClient.class);
|
||||
|
||||
|
||||
public Host getDataminer(String hostname) ***REMOVED***
|
||||
|
|
@ -160,6 +173,130 @@ public class ISClient ***REMOVED***
|
|||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public void updateAlg(Algorithm algo) ***REMOVED***
|
||||
|
||||
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
||||
|
||||
SimpleQuery query = queryFor(GenericResource.class);
|
||||
query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource");
|
||||
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
|
||||
List<GenericResource> ds = client.submit(query);
|
||||
if (ds.isEmpty()) ***REMOVED***
|
||||
return;
|
||||
***REMOVED***
|
||||
GenericResource a = ds.get(0);
|
||||
a.profile().newBody(this.getAlgoBody(algo));
|
||||
try ***REMOVED***
|
||||
scopedPublisher.update(a);
|
||||
***REMOVED*** catch (RegistryNotFoundException e) ***REMOVED***
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
private String getAlgoBody(Algorithm algo) ***REMOVED***
|
||||
return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
|
||||
+ "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
|
||||
+ algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
|
||||
+ "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED*** public void addAlgToIs(Algorithm algo) ***REMOVED***
|
||||
***REMOVED*** GenericResource a = new GenericResource();
|
||||
***REMOVED*** a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
|
||||
***REMOVED*** a.profile().newBody(this.getAlgoBody(algo));
|
||||
***REMOVED*** try ***REMOVED***
|
||||
***REMOVED*** publishScopedResource(a, Arrays.asList(new String[] ***REMOVED*** ScopeProvider.instance.get() ***REMOVED***));
|
||||
***REMOVED*** ***REMOVED*** catch (Exception e) ***REMOVED***
|
||||
***REMOVED*** e.printStackTrace();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
|
||||
|
||||
|
||||
public void addAlgToIs(Algorithm algo, String token) ***REMOVED***
|
||||
GenericResource a = new GenericResource();
|
||||
a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
|
||||
a.profile().newBody(this.getAlgoBody(algo));
|
||||
try ***REMOVED***
|
||||
SecurityTokenProvider.instance.set(token);
|
||||
publishScopedResource(a, Arrays.asList(new String[] ***REMOVED*** ScopeProvider.instance.get() ***REMOVED***));
|
||||
***REMOVED*** catch (Exception e) ***REMOVED***
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception ***REMOVED***
|
||||
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
||||
AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher);
|
||||
String id = resource.id();
|
||||
LOGGER.debug("Trying to remove ***REMOVED******REMOVED*** with ID ***REMOVED******REMOVED*** from ***REMOVED******REMOVED***", resource.getClass().getSimpleName(), id,
|
||||
ScopeProvider.instance.get());
|
||||
***REMOVED*** scopedPublisher.remove(resource, scopes);
|
||||
advancedScopedPublisher.forceRemove(resource);
|
||||
LOGGER.debug("***REMOVED******REMOVED*** with ID ***REMOVED******REMOVED*** removed successfully", resource.getClass().getSimpleName(), id);
|
||||
***REMOVED***
|
||||
|
||||
public void publishScopedResource(GenericResource a, List<String> scopes)
|
||||
throws RegistryNotFoundException, Exception ***REMOVED***
|
||||
StringWriter stringWriter = new StringWriter();
|
||||
Resources.marshal(a, stringWriter);
|
||||
|
||||
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
||||
try ***REMOVED***
|
||||
System.out.println(scopes);
|
||||
System.out.println(stringWriter);
|
||||
scopedPublisher.create(a, scopes);
|
||||
***REMOVED*** catch (RegistryNotFoundException e) ***REMOVED***
|
||||
System.out.println(e);
|
||||
throw e;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public Set<Algorithm> getAlgoFromIs() ***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
Set<Algorithm> out = new HashSet<Algorithm>();
|
||||
SimpleQuery query = queryFor(GenericResource.class);
|
||||
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
|
||||
.setResult("$resource");
|
||||
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
|
||||
List<GenericResource> ds = client.submit(query);
|
||||
for (GenericResource a : ds) ***REMOVED***
|
||||
out.add(this.convertAlgo(a));
|
||||
***REMOVED***
|
||||
return out;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
private Algorithm convertAlgo(GenericResource a) ***REMOVED***
|
||||
Algorithm out = new Algorithm();
|
||||
|
||||
***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
|
||||
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
|
||||
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
|
||||
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
|
||||
out.setName(a.profile().name());
|
||||
out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent());
|
||||
out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent());
|
||||
out.setDescription(a.profile().description());
|
||||
|
||||
Set<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency> deps = new HashSet<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency>();
|
||||
for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) ***REMOVED***
|
||||
org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency();
|
||||
d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent());
|
||||
deps.add(d1);
|
||||
***REMOVED***
|
||||
out.setDependencies(deps);
|
||||
return out;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public static void main(String[] args) throws IOException, SVNException ***REMOVED***
|
||||
ISClient a = new ISClient();
|
||||
ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import org.gcube.common.resources.gcore.Resources;
|
|||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
|
||||
|
|
@ -31,7 +32,6 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
|||
|
||||
public class DataminerPoolManager ***REMOVED***
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
||||
|
||||
|
||||
private SVNUpdater svnUpdater;
|
||||
|
|
@ -68,15 +68,30 @@ public class DataminerPoolManager ***REMOVED***
|
|||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
***REMOVED***1. to complete
|
||||
private Cluster getStagingDataminerCluster()***REMOVED***
|
||||
Cluster cluster = new Cluster();
|
||||
Host h = new Host();
|
||||
***REMOVED***TODO: read this from configuration
|
||||
h.setName("test");
|
||||
***REMOVED***TODO: read this from configuration or IS?
|
||||
h.setName("dataminer1-devnext.d4science.org");
|
||||
cluster.addHost(h);
|
||||
return cluster;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
private void createISResource(Algorithm algo, String vreToken) ***REMOVED***
|
||||
ISClient client = new ISClient();
|
||||
for (Algorithm a : client.getAlgoFromIs()) ***REMOVED***
|
||||
if (a.getName().equals(algo.getName())) ***REMOVED***
|
||||
continue;
|
||||
***REMOVED***
|
||||
if (!a.getName().equals(algo.getName()))***REMOVED***
|
||||
new ISClient().addAlgToIs(algo, vreToken);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** TODO: create the resource only if not already present
|
||||
***REMOVED***
|
||||
|
||||
|
||||
private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED***
|
||||
for (Dependency d : algo.getDependencies()) ***REMOVED***
|
||||
|
|
@ -100,14 +115,6 @@ public class DataminerPoolManager ***REMOVED***
|
|||
***REMOVED***
|
||||
|
||||
|
||||
private void createISResource(Algorithm algo, String vreToken)***REMOVED***
|
||||
***REMOVED***TODO: implement method
|
||||
|
||||
***REMOVED***TODO: create the resource only if not already present
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED****
|
||||
*
|
||||
* @param algo
|
||||
|
|
@ -297,109 +304,4 @@ public class DataminerPoolManager ***REMOVED***
|
|||
***REMOVED******REMOVED***
|
||||
|
||||
|
||||
***REMOVED*** 2017 March 29
|
||||
|
||||
public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception ***REMOVED***
|
||||
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
||||
AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher);
|
||||
String id = resource.id();
|
||||
LOGGER.debug("Trying to remove ***REMOVED******REMOVED*** with ID ***REMOVED******REMOVED*** from ***REMOVED******REMOVED***", resource.getClass().getSimpleName(), id,
|
||||
ScopeProvider.instance.get());
|
||||
***REMOVED*** scopedPublisher.remove(resource, scopes);
|
||||
advancedScopedPublisher.forceRemove(resource);
|
||||
LOGGER.debug("***REMOVED******REMOVED*** with ID ***REMOVED******REMOVED*** removed successfully", resource.getClass().getSimpleName(), id);
|
||||
***REMOVED***
|
||||
|
||||
public void publishScopedResource(GenericResource a, List<String> scopes)
|
||||
throws RegistryNotFoundException, Exception ***REMOVED***
|
||||
StringWriter stringWriter = new StringWriter();
|
||||
Resources.marshal(a, stringWriter);
|
||||
|
||||
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
||||
try ***REMOVED***
|
||||
System.out.println(scopes);
|
||||
System.out.println(stringWriter);
|
||||
scopedPublisher.create(a, scopes);
|
||||
***REMOVED*** catch (RegistryNotFoundException e) ***REMOVED***
|
||||
System.out.println(e);
|
||||
throw e;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public void addAlgToIs(Algorithm algo) ***REMOVED***
|
||||
GenericResource a = new GenericResource();
|
||||
a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
|
||||
a.profile().newBody(this.getAlgoBody(algo));
|
||||
try ***REMOVED***
|
||||
publishScopedResource(a, Arrays.asList(new String[] ***REMOVED*** ScopeProvider.instance.get() ***REMOVED***));
|
||||
***REMOVED*** catch (Exception e) ***REMOVED***
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
private String getAlgoBody(Algorithm algo) ***REMOVED***
|
||||
return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
|
||||
+ "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
|
||||
+ algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
|
||||
+ "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
|
||||
***REMOVED***
|
||||
|
||||
public void updateAlg(Algorithm algo) ***REMOVED***
|
||||
|
||||
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
|
||||
|
||||
SimpleQuery query = queryFor(GenericResource.class);
|
||||
query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource");
|
||||
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
|
||||
List<GenericResource> ds = client.submit(query);
|
||||
if (ds.isEmpty()) ***REMOVED***
|
||||
return;
|
||||
***REMOVED***
|
||||
GenericResource a = ds.get(0);
|
||||
a.profile().newBody(this.getAlgoBody(algo));
|
||||
try ***REMOVED***
|
||||
scopedPublisher.update(a);
|
||||
***REMOVED*** catch (RegistryNotFoundException e) ***REMOVED***
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
private Algorithm convertAlgo(GenericResource a) ***REMOVED***
|
||||
Algorithm out = new Algorithm();
|
||||
|
||||
***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
|
||||
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
|
||||
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
|
||||
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
|
||||
out.setName(a.profile().name());
|
||||
out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent());
|
||||
out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent());
|
||||
out.setDescription(a.profile().description());
|
||||
|
||||
Set<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency> deps = new HashSet<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency>();
|
||||
for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) ***REMOVED***
|
||||
org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency();
|
||||
d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent());
|
||||
deps.add(d1);
|
||||
***REMOVED***
|
||||
out.setDependencies(deps);
|
||||
return out;
|
||||
***REMOVED***
|
||||
|
||||
public Set<Algorithm> getAlgoFromIs() ***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
Set<Algorithm> out = new HashSet<Algorithm>();
|
||||
SimpleQuery query = queryFor(GenericResource.class);
|
||||
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
|
||||
.setResult("$resource");
|
||||
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
|
||||
List<GenericResource> ds = client.submit(query);
|
||||
for (GenericResource a : ds) ***REMOVED***
|
||||
out.add(this.convertAlgo(a));
|
||||
***REMOVED***
|
||||
return out;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
Loading…
Reference in New Issue