This commit is contained in:
Nunzio Andrea Galante 2017-05-11 10:58:38 +00:00
parent 9d3cf1deb6
commit 3331ec29b2
2 changed files with 156 additions and 117 deletions

View File

@ -10,6 +10,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
@ -27,15 +28,26 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.Vector;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.common.resources.gcore.Resources;
import org.gcube.common.resources.gcore.ServiceEndpoint;
import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
import org.gcube.informationsystem.publisher.ScopedPublisher;
import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNDepth;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL;
@ -52,6 +64,7 @@ import org.tmatesoft.svn.core.wc.SVNWCUtil;
import au.com.bytecode.opencsv.CSVReader;
public class ISClient {
private static final Logger LOGGER = LoggerFactory.getLogger(ISClient.class);
public Host getDataminer(String hostname) {
@ -160,6 +173,130 @@ public class ISClient {
}
}
public void updateAlg(Algorithm algo) {
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
if (ds.isEmpty()) {
return;
}
GenericResource a = ds.get(0);
a.profile().newBody(this.getAlgoBody(algo));
try {
scopedPublisher.update(a);
} catch (RegistryNotFoundException e) {
e.printStackTrace();
}
}
private String getAlgoBody(Algorithm algo) {
return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
+ "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
+ algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
+ "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
}
// public void addAlgToIs(Algorithm algo) {
// GenericResource a = new GenericResource();
// a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
// a.profile().newBody(this.getAlgoBody(algo));
// try {
// publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() }));
// } catch (Exception e) {
// e.printStackTrace();
// }
// }
public void addAlgToIs(Algorithm algo, String token) {
GenericResource a = new GenericResource();
a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
a.profile().newBody(this.getAlgoBody(algo));
try {
SecurityTokenProvider.instance.set(token);
publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() }));
} catch (Exception e) {
e.printStackTrace();
}
}
public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception {
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher);
String id = resource.id();
LOGGER.debug("Trying to remove {} with ID {} from {}", resource.getClass().getSimpleName(), id,
ScopeProvider.instance.get());
// scopedPublisher.remove(resource, scopes);
advancedScopedPublisher.forceRemove(resource);
LOGGER.debug("{} with ID {} removed successfully", resource.getClass().getSimpleName(), id);
}
public void publishScopedResource(GenericResource a, List<String> scopes)
throws RegistryNotFoundException, Exception {
StringWriter stringWriter = new StringWriter();
Resources.marshal(a, stringWriter);
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
try {
System.out.println(scopes);
System.out.println(stringWriter);
scopedPublisher.create(a, scopes);
} catch (RegistryNotFoundException e) {
System.out.println(e);
throw e;
}
}
public Set<Algorithm> getAlgoFromIs() {
// TODO Auto-generated method stub
Set<Algorithm> out = new HashSet<Algorithm>();
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
.setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
for (GenericResource a : ds) {
out.add(this.convertAlgo(a));
}
return out;
}
private Algorithm convertAlgo(GenericResource a) {
Algorithm out = new Algorithm();
// out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
out.setName(a.profile().name());
out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent());
out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent());
out.setDescription(a.profile().description());
Set<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency> deps = new HashSet<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency>();
for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) {
org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency();
d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent());
deps.add(d1);
}
out.setDependencies(deps);
return out;
}
public static void main(String[] args) throws IOException, SVNException {
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");

View File

@ -7,6 +7,7 @@ import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
@ -31,7 +32,6 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
public class DataminerPoolManager {
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
private SVNUpdater svnUpdater;
@ -68,15 +68,30 @@ public class DataminerPoolManager {
}
//1. to complete
private Cluster getStagingDataminerCluster(){
Cluster cluster = new Cluster();
Host h = new Host();
//TODO: read this from configuration
h.setName("test");
//TODO: read this from configuration or IS?
h.setName("dataminer1-devnext.d4science.org");
cluster.addHost(h);
return cluster;
}
private void createISResource(Algorithm algo, String vreToken) {
ISClient client = new ISClient();
for (Algorithm a : client.getAlgoFromIs()) {
if (a.getName().equals(algo.getName())) {
continue;
}
if (!a.getName().equals(algo.getName())){
new ISClient().addAlgToIs(algo, vreToken);
}
}
// TODO: create the resource only if not already present
}
private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
for (Dependency d : algo.getDependencies()) {
@ -100,14 +115,6 @@ public class DataminerPoolManager {
}
private void createISResource(Algorithm algo, String vreToken){
//TODO: implement method
//TODO: create the resource only if not already present
}
/**
*
* @param algo
@ -297,109 +304,4 @@ public class DataminerPoolManager {
// }
// 2017 March 29
public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception {
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher);
String id = resource.id();
LOGGER.debug("Trying to remove {} with ID {} from {}", resource.getClass().getSimpleName(), id,
ScopeProvider.instance.get());
// scopedPublisher.remove(resource, scopes);
advancedScopedPublisher.forceRemove(resource);
LOGGER.debug("{} with ID {} removed successfully", resource.getClass().getSimpleName(), id);
}
public void publishScopedResource(GenericResource a, List<String> scopes)
throws RegistryNotFoundException, Exception {
StringWriter stringWriter = new StringWriter();
Resources.marshal(a, stringWriter);
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
try {
System.out.println(scopes);
System.out.println(stringWriter);
scopedPublisher.create(a, scopes);
} catch (RegistryNotFoundException e) {
System.out.println(e);
throw e;
}
}
public void addAlgToIs(Algorithm algo) {
GenericResource a = new GenericResource();
a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription());
a.profile().newBody(this.getAlgoBody(algo));
try {
publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() }));
} catch (Exception e) {
e.printStackTrace();
}
}
private String getAlgoBody(Algorithm algo) {
return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
+ "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
+ algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
+ "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
}
public void updateAlg(Algorithm algo) {
ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher();
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
if (ds.isEmpty()) {
return;
}
GenericResource a = ds.get(0);
a.profile().newBody(this.getAlgoBody(algo));
try {
scopedPublisher.update(a);
} catch (RegistryNotFoundException e) {
e.printStackTrace();
}
}
private Algorithm convertAlgo(GenericResource a) {
Algorithm out = new Algorithm();
// out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
out.setName(a.profile().name());
out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent());
out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent());
out.setDescription(a.profile().description());
Set<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency> deps = new HashSet<org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency>();
for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) {
org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency();
d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent());
deps.add(d1);
}
out.setDependencies(deps);
return out;
}
public Set<Algorithm> getAlgoFromIs() {
// TODO Auto-generated method stub
Set<Algorithm> out = new HashSet<Algorithm>();
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
.setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
for (GenericResource a : ds) {
out.add(this.convertAlgo(a));
}
return out;
}
}