diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index 5acd531..044103e 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -10,6 +10,7 @@ import java.io.File; import java.io.InputStream; ***REMOVED*** import java.io.PrintStream; +import java.io.StringWriter; import java.net.MalformedURLException; ***REMOVED*** import java.util.ArrayList; @@ -27,15 +28,26 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.Vector; +***REMOVED*** +import org.gcube.common.resources.gcore.GenericResource; +import org.gcube.common.resources.gcore.Resources; import org.gcube.common.resources.gcore.ServiceEndpoint; ***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; +import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; +import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; +import org.gcube.informationsystem.publisher.RegistryPublisherFactory; +import org.gcube.informationsystem.publisher.ScopedPublisher; +import org.gcube.informationsystem.publisher.exception.RegistryNotFoundException; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +***REMOVED*** +***REMOVED*** import org.tmatesoft.svn.core.SVNDepth; import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNURL; @@ -52,6 +64,7 @@ import org.tmatesoft.svn.core.wc.SVNWCUtil; ***REMOVED*** public class ISClient { + private static final Logger LOGGER = LoggerFactory.getLogger(ISClient.class); public Host getDataminer(String hostname) { @@ -160,6 +173,130 @@ public class ISClient { ***REMOVED*** ***REMOVED*** + + public void updateAlg(Algorithm algo) { + + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + + SimpleQuery query = queryFor(GenericResource.class); + query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource"); + DiscoveryClient client = clientFor(GenericResource.class); + List ds = client.submit(query); + if (ds.isEmpty()) { + return; + ***REMOVED*** + GenericResource a = ds.get(0); + a.profile().newBody(this.getAlgoBody(algo)); + try { + scopedPublisher.update(a); + ***REMOVED*** catch (RegistryNotFoundException e) { + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + + + private String getAlgoBody(Algorithm algo) { + return "" + algo.getCategory() + "" + "\n" + "" + algo.getClazz() + "" + + "\n" + "" + algo.getAlgorithmType() + "" + "\n" + "" + + algo.getSkipJava() + "" + "\n" + "" + algo.getPackageURL() + "" + + "\n" + "" + algo.getDependencies() + ""; +***REMOVED*** + + + +***REMOVED*** public void addAlgToIs(Algorithm algo) { +***REMOVED*** GenericResource a = new GenericResource(); +***REMOVED*** a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); +***REMOVED*** a.profile().newBody(this.getAlgoBody(algo)); +***REMOVED*** try { +***REMOVED*** publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() ***REMOVED***)); +***REMOVED*** ***REMOVED*** catch (Exception e) { +***REMOVED*** e.printStackTrace(); +***REMOVED*** ***REMOVED*** +***REMOVED******REMOVED*** + + + + public void addAlgToIs(Algorithm algo, String token) { + GenericResource a = new GenericResource(); + a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); + a.profile().newBody(this.getAlgoBody(algo)); + try { + SecurityTokenProvider.instance.set(token); + publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() ***REMOVED***)); + ***REMOVED*** catch (Exception e) { + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + + public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception { + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher); + String id = resource.id(); + LOGGER.debug("Trying to remove {***REMOVED*** with ID {***REMOVED*** from {***REMOVED***", resource.getClass().getSimpleName(), id, + ScopeProvider.instance.get()); + ***REMOVED*** scopedPublisher.remove(resource, scopes); + advancedScopedPublisher.forceRemove(resource); + LOGGER.debug("{***REMOVED*** with ID {***REMOVED*** removed successfully", resource.getClass().getSimpleName(), id); +***REMOVED*** + + public void publishScopedResource(GenericResource a, List scopes) + throws RegistryNotFoundException, Exception { + StringWriter stringWriter = new StringWriter(); + Resources.marshal(a, stringWriter); + + ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); + try { + System.out.println(scopes); + System.out.println(stringWriter); + scopedPublisher.create(a, scopes); + ***REMOVED*** catch (RegistryNotFoundException e) { + System.out.println(e); + throw e; + ***REMOVED*** +***REMOVED*** + + + public Set getAlgoFromIs() { + ***REMOVED*** TODO Auto-generated method stub + + Set out = new HashSet(); + SimpleQuery query = queryFor(GenericResource.class); + query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'") + .setResult("$resource"); + DiscoveryClient client = clientFor(GenericResource.class); + List ds = client.submit(query); + for (GenericResource a : ds) { + out.add(this.convertAlgo(a)); + ***REMOVED*** +***REMOVED*** +***REMOVED*** + + + private Algorithm convertAlgo(GenericResource a) { + Algorithm out = new Algorithm(); + + ***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); + out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); + out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); + out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); + out.setName(a.profile().name()); + out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); + out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); + out.setDescription(a.profile().description()); + + Set deps = new HashSet(); + for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) { + org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); + d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); + deps.add(d1); + ***REMOVED*** + out.setDependencies(deps); +***REMOVED*** +***REMOVED*** + + + public static void main(String[] args) throws IOException, SVNException { ISClient a = new ISClient(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 6b51f72..c4387a5 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -7,6 +7,7 @@ import org.gcube.common.resources.gcore.Resources; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; +import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; @@ -31,7 +32,6 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; public class DataminerPoolManager { - private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); private SVNUpdater svnUpdater; @@ -68,15 +68,30 @@ public class DataminerPoolManager { ***REMOVED*** - + ***REMOVED***1. to complete private Cluster getStagingDataminerCluster(){ Cluster cluster = new Cluster(); Host h = new Host(); - ***REMOVED***TODO: read this from configuration - h.setName("test"); + ***REMOVED***TODO: read this from configuration or IS? + h.setName("dataminer1-devnext.d4science.org"); cluster.addHost(h); return cluster; ***REMOVED*** + + + private void createISResource(Algorithm algo, String vreToken) { + ISClient client = new ISClient(); + for (Algorithm a : client.getAlgoFromIs()) { + if (a.getName().equals(algo.getName())) { + continue; + ***REMOVED*** + if (!a.getName().equals(algo.getName())){ + new ISClient().addAlgToIs(algo, vreToken); + ***REMOVED*** + ***REMOVED*** + ***REMOVED*** TODO: create the resource only if not already present +***REMOVED*** + private void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException { for (Dependency d : algo.getDependencies()) { @@ -100,14 +115,6 @@ public class DataminerPoolManager { ***REMOVED*** - private void createISResource(Algorithm algo, String vreToken){ - ***REMOVED***TODO: implement method - - ***REMOVED***TODO: create the resource only if not already present -***REMOVED*** - - - /** * * @param algo @@ -297,109 +304,4 @@ public class DataminerPoolManager { ***REMOVED******REMOVED*** - ***REMOVED*** 2017 March 29 - - public void unPublishScopedResource(GenericResource resource) throws RegistryNotFoundException, Exception { - ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); - AdvancedScopedPublisher advancedScopedPublisher = new AdvancedScopedPublisher(scopedPublisher); - String id = resource.id(); - LOGGER.debug("Trying to remove {***REMOVED*** with ID {***REMOVED*** from {***REMOVED***", resource.getClass().getSimpleName(), id, - ScopeProvider.instance.get()); - ***REMOVED*** scopedPublisher.remove(resource, scopes); - advancedScopedPublisher.forceRemove(resource); - LOGGER.debug("{***REMOVED*** with ID {***REMOVED*** removed successfully", resource.getClass().getSimpleName(), id); -***REMOVED*** - - public void publishScopedResource(GenericResource a, List scopes) - throws RegistryNotFoundException, Exception { - StringWriter stringWriter = new StringWriter(); - Resources.marshal(a, stringWriter); - - ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); - try { - System.out.println(scopes); - System.out.println(stringWriter); - scopedPublisher.create(a, scopes); - ***REMOVED*** catch (RegistryNotFoundException e) { - System.out.println(e); - throw e; - ***REMOVED*** -***REMOVED*** - - public void addAlgToIs(Algorithm algo) { - GenericResource a = new GenericResource(); - a.newProfile().name(algo.getName()).type("StatisticalManagerAlgorithm").description(algo.getDescription()); - a.profile().newBody(this.getAlgoBody(algo)); - try { - publishScopedResource(a, Arrays.asList(new String[] { ScopeProvider.instance.get() ***REMOVED***)); - ***REMOVED*** catch (Exception e) { - e.printStackTrace(); - ***REMOVED*** -***REMOVED*** - - private String getAlgoBody(Algorithm algo) { - return "" + algo.getCategory() + "" + "\n" + "" + algo.getClazz() + "" - + "\n" + "" + algo.getAlgorithmType() + "" + "\n" + "" - + algo.getSkipJava() + "" + "\n" + "" + algo.getPackageURL() + "" - + "\n" + "" + algo.getDependencies() + ""; -***REMOVED*** - - public void updateAlg(Algorithm algo) { - - ScopedPublisher scopedPublisher = RegistryPublisherFactory.scopedPublisher(); - - SimpleQuery query = queryFor(GenericResource.class); - query.addCondition("$resource/Profile/Name/text() eq '" + algo.getName() + "'").setResult("$resource"); - DiscoveryClient client = clientFor(GenericResource.class); - List ds = client.submit(query); - if (ds.isEmpty()) { - return; - ***REMOVED*** - GenericResource a = ds.get(0); - a.profile().newBody(this.getAlgoBody(algo)); - try { - scopedPublisher.update(a); - ***REMOVED*** catch (RegistryNotFoundException e) { - e.printStackTrace(); - ***REMOVED*** -***REMOVED*** - - private Algorithm convertAlgo(GenericResource a) { - Algorithm out = new Algorithm(); - - ***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); - out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); - out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); - out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); - out.setName(a.profile().name()); - out.setPackageURL(a.profile().body().getElementsByTagName("packageURL").item(0).getTextContent()); - out.setSkipJava(a.profile().body().getElementsByTagName("skipJava").item(0).getTextContent()); - out.setDescription(a.profile().description()); - - Set deps = new HashSet(); - for (int i = 0; i < a.profile().body().getElementsByTagName("dependencies").getLength(); i++) { - org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency d1 = new org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency(); - d1.setName(a.profile().body().getElementsByTagName("dependencies").item(i).getTextContent()); - deps.add(d1); - ***REMOVED*** - out.setDependencies(deps); -***REMOVED*** -***REMOVED*** - - public Set getAlgoFromIs() { - ***REMOVED*** TODO Auto-generated method stub - - Set out = new HashSet(); - SimpleQuery query = queryFor(GenericResource.class); - query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'") - .setResult("$resource"); - DiscoveryClient client = clientFor(GenericResource.class); - List ds = client.submit(query); - for (GenericResource a : ds) { - out.add(this.convertAlgo(a)); - ***REMOVED*** -***REMOVED*** -***REMOVED*** - - ***REMOVED***