diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index 86fa481..a24838c 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -10,6 +10,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; +import java.io.StringWriter; import java.net.MalformedURLException; import java.net.URL; import java.util.ArrayList; @@ -27,15 +28,26 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.Vector; +import org.gcube.common.authorization.library.provider.SecurityTokenProvider; +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.Resources; import org.gcube.common.resources.gcore.ServiceEndpoint; import org.gcube.common.scope.api.ScopeProvider; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; +import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; +import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; +import org.gcube.informationsystem.publisher.RegistryPublisherFactory; +import org.gcube.informationsystem.publisher.ScopedPublisher; +import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.tmatesoft.svn.core.SVNDepth; import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNURL; @@ -52,6 +64,7 @@ import org.tmatesoft.svn.core.wc.SVNWCUtil; import au.com.bytecode.opencsv.CSVReader; public class ISClient { + private static final Logger LOGGER = LoggerFactory.getLogger(ISClient.class); public Host getDataminer(String hostname) { @@ -160,6 +173,130 @@ public class ISClient { } } + + public void updateAlg(Algorithm algo) { + + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + + SimpleQuery query = queryFor(GenericResource.class); + query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource"); + DiscoveryClient client = clientFor(GenericResource.class); + List ds = client.submit(query); + if (ds.isEmpty()) { + return; + } + GenericResource a = ds.get(0); + a.profile().newBody(this.getAlgoBody(algo)); + try { + scopedPublisher.update(a); + } catch (RegistryNotFoundException e) { + e.printStackTrace(); + } + } + + + private String getAlgoBody(Algorithm algo) { + return "" + algo.getCategory() + "" + "\n" + "" + algo.getClazz() + "" + + "\n" + "" + algo.getAlgorithmType() + "" + "\n" + "" + + algo.getSkipJava() + "" + "\n" + "" + algo.getPackageURL() + "" + + "\n" + "" + algo.getDependencies() + ""; + } + + + +// public void addAlgToIs(Algorithm algo) { +// GenericResource a = new GenericResource(); +// a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); +// a.profile().newBody(this.getAlgoBody(algo)); +// try { +// publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() })); +// } catch (Exception e) { +// e.printStackTrace(); +// } +// } + + + + public void addAlgToIs(Algorithm algo, String token) { + GenericResource a = new GenericResource(); + a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); + a.profile().newBody(this.getAlgoBody(algo)); + try { + SecurityTokenProvider.instance.set(token); + publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() })); + } catch (Exception e) { + e.printStackTrace(); + } + } + + public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception { + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher); + String id = resource.id(); + LOGGER.debug("Trying to remove {} with ID {} from {}", resource.getClass().getSimpleName(), id, + ScopeProvider.instance.get()); + // scopedPublisher.remove(resource, scopes); + advancedScopedPublisher.forceRemove(resource); + LOGGER.debug("{} with ID {} removed successfully", resource.getClass().getSimpleName(), id); + } + + public void publishScopedResource(GenericResource a, List scopes) + throws RegistryNotFoundException, Exception { + StringWriter stringWriter = new StringWriter(); + Resources.marshal(a, stringWriter); + + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + try { + System.out.println(scopes); + System.out.println(stringWriter); + scopedPublisher.create(a, scopes); + } catch (RegistryNotFoundException e) { + System.out.println(e); + throw e; + } + } + + + public Set getAlgoFromIs() { + // TODO Auto-generated method stub + + Set out = new HashSet(); + SimpleQuery query = queryFor(GenericResource.class); + query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'") + .setResult("$resource"); + DiscoveryClient client = clientFor(GenericResource.class); + List ds = client.submit(query); + for (GenericResource a : ds) { + out.add(this.convertAlgo(a)); + } + return out; + } + + + private Algorithm convertAlgo(GenericResource a) { + Algorithm out = new Algorithm(); + + // out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); + out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); + out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); + out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); + out.setName(a.profile().name()); + out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); + out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); + out.setDescription(a.profile().description()); + + Set deps = new HashSet(); + for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) { + org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); + d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); + deps.add(d1); + } + out.setDependencies(deps); + return out; + } + + + public static void main(String[] args) throws IOException, SVNException { ISClient a = new ISClient(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 63af652..909fa42 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -7,6 +7,7 @@ import org.gcube.common.scope.api.ScopeProvider; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; +import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; @@ -31,7 +32,6 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; public class DataminerPoolManager { - private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); private SVNUpdater svnUpdater; @@ -68,15 +68,30 @@ public class DataminerPoolManager { } - + //1. to complete private Cluster getStagingDataminerCluster(){ Cluster cluster = new Cluster(); Host h = new Host(); - //TODO: read this from configuration - h.setName("test"); + //TODO: read this from configuration or IS? + h.setName("dataminer1-devnext.d4science.org"); cluster.addHost(h); return cluster; } + + + private void createISResource(Algorithm algo, String vreToken) { + ISClient client = new ISClient(); + for (Algorithm a : client.getAlgoFromIs()) { + if (a.getName().equals(algo.getName())) { + continue; + } + if (!a.getName().equals(algo.getName())){ + new ISClient().addAlgToIs(algo, vreToken); + } + } + // TODO: create the resource only if not already present + } + private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException { for (Dependency d : algo.getDependencies()) { @@ -100,14 +115,6 @@ public class DataminerPoolManager { } - private void createISResource(Algorithm algo, String vreToken){ - //TODO: implement method - - //TODO: create the resource only if not already present - } - - - /** * * @param algo @@ -297,109 +304,4 @@ public class DataminerPoolManager { // } - // 2017 March 29 - - public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception { - ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); - AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher); - String id = resource.id(); - LOGGER.debug("Trying to remove {} with ID {} from {}", resource.getClass().getSimpleName(), id, - ScopeProvider.instance.get()); - // scopedPublisher.remove(resource, scopes); - advancedScopedPublisher.forceRemove(resource); - LOGGER.debug("{} with ID {} removed successfully", resource.getClass().getSimpleName(), id); - } - - public void publishScopedResource(GenericResource a, List scopes) - throws RegistryNotFoundException, Exception { - StringWriter stringWriter = new StringWriter(); - Resources.marshal(a, stringWriter); - - ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); - try { - System.out.println(scopes); - System.out.println(stringWriter); - scopedPublisher.create(a, scopes); - } catch (RegistryNotFoundException e) { - System.out.println(e); - throw e; - } - } - - public void addAlgToIs(Algorithm algo) { - GenericResource a = new GenericResource(); - a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); - a.profile().newBody(this.getAlgoBody(algo)); - try { - publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() })); - } catch (Exception e) { - e.printStackTrace(); - } - } - - private String getAlgoBody(Algorithm algo) { - return "" + algo.getCategory() + "" + "\n" + "" + algo.getClazz() + "" - + "\n" + "" + algo.getAlgorithmType() + "" + "\n" + "" - + algo.getSkipJava() + "" + "\n" + "" + algo.getPackageURL() + "" - + "\n" + "" + algo.getDependencies() + ""; - } - - public void updateAlg(Algorithm algo) { - - ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); - - SimpleQuery query = queryFor(GenericResource.class); - query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource"); - DiscoveryClient client = clientFor(GenericResource.class); - List ds = client.submit(query); - if (ds.isEmpty()) { - return; - } - GenericResource a = ds.get(0); - a.profile().newBody(this.getAlgoBody(algo)); - try { - scopedPublisher.update(a); - } catch (RegistryNotFoundException e) { - e.printStackTrace(); - } - } - - private Algorithm convertAlgo(GenericResource a) { - Algorithm out = new Algorithm(); - - // out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); - out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); - out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); - out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); - out.setName(a.profile().name()); - out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); - out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); - out.setDescription(a.profile().description()); - - Set deps = new HashSet(); - for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) { - org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); - d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); - deps.add(d1); - } - out.setDependencies(deps); - return out; - } - - public Set getAlgoFromIs() { - // TODO Auto-generated method stub - - Set out = new HashSet(); - SimpleQuery query = queryFor(GenericResource.class); - query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'") - .setResult("$resource"); - DiscoveryClient client = clientFor(GenericResource.class); - List ds = client.submit(query); - for (GenericResource a : ds) { - out.add(this.convertAlgo(a)); - } - return out; - } - - }