From 5018f510900782d08b3d02af5d3c2272c2d94a02 Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Wed, 5 Apr 2017 15:14:18 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146587 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 5 + .../poolmanager/clients/ISClient.java | 43 ++++++++- .../poolmanager/rest/PoolManager.java | 3 + .../poolmanager/rest/RestPoolManager.java | 96 +++++++++++++++++-- .../service/DataminerPoolManager.java | 66 +++++++++++++ 5 files changed, 202 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index bc65c60..d36500c 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,11 @@ [4.0.0, 4.1.0) + org.tmatesoft.svnkit + svnkit + 1.8.5 + + commons-configuration commons-configuration 1.10 diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index 1a20631..ffdd57e 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -70,15 +70,24 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; ***REMOVED*** +import java.io.ByteArrayOutputStream; +import java.io.File; ***REMOVED*** +import java.io.InputStream; ***REMOVED*** +import java.io.PrintStream; import java.net.MalformedURLException; ***REMOVED*** +import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.LinkedHashMap; ***REMOVED*** ***REMOVED*** import java.util.Map; +import java.util.Scanner; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; @@ -87,11 +96,24 @@ import java.util.Vector; import org.gcube.common.resources.gcore.ServiceEndpoint; ***REMOVED*** ***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; +import org.tmatesoft.svn.core.SVNDepth; +import org.tmatesoft.svn.core.SVNException; +import org.tmatesoft.svn.core.SVNURL; +import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; +import org.tmatesoft.svn.core.internal.io.dav.DAVRepositoryFactory; +import org.tmatesoft.svn.core.io.SVNRepository; +import org.tmatesoft.svn.core.io.SVNRepositoryFactory; +import org.tmatesoft.svn.core.wc.ISVNOptions; +import org.tmatesoft.svn.core.wc.SVNClientManager; +import org.tmatesoft.svn.core.wc.SVNRevision; +import org.tmatesoft.svn.core.wc.SVNUpdateClient; +import org.tmatesoft.svn.core.wc.SVNWCUtil; ***REMOVED*** @@ -168,6 +190,8 @@ public Cluster getClusterByHProxy() throws IOException { ***REMOVED*** + + /** * Return the list of hosts (dataminers) in a given VRE * @@ -210,12 +234,25 @@ public Cluster getClusterByHProxy() throws IOException { ***REMOVED*** ***REMOVED*** - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException, SVNException { ISClient a = new ISClient(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); ***REMOVED***System.out.println(a.getHProxy()); - System.out.println(a.getClusterByHProxy()); - System.out.println(a.listDataMinersByCluster()); + ***REMOVED***System.out.println(a.getClusterByHProxy()); + ***REMOVED***System.out.println(a.listDataMinersByCluster()); + + + ***REMOVED***List list = new LinkedList(); + ***REMOVED***Dependency aa = new Dependency(); + ***REMOVED***aa.setName("testnunzio"); + ***REMOVED***aa.setType("cran:"); + ***REMOVED***list.add(aa); + + + + + + ***REMOVED***a.checkSVNdep(); ***REMOVED***System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain()); ***REMOVED***System.out.println(a.listDataminersInVRE()); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index 12643cf..d7fb0d1 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -31,9 +31,11 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest; import java.net.MalformedURLException; ***REMOVED*** import java.net.UnknownHostException; +***REMOVED*** import java.util.Set; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.tmatesoft.svn.core.SVNException; public interface PoolManager { @@ -53,6 +55,7 @@ public interface PoolManager { void addAlgToIs(Algorithm algo); Set getAlgoFromIs(); + List getOriginalContents(String file, List ldep) throws SVNException; ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index d3eafe6..9b0cb10 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -142,11 +142,14 @@ ***REMOVED******REMOVED*** package org.gcube.dataanalysis.dataminer.poolmanager.rest; +import java.awt.color.ICC_ColorSpace; ***REMOVED*** import java.net.MalformedURLException; import java.net.ProxySelector; ***REMOVED*** import java.net.UnknownHostException; +***REMOVED*** +***REMOVED*** import java.util.Random; import java.util.Set; import java.util.UUID; @@ -159,11 +162,14 @@ import javax.ws.rs.QueryParam; import org.apache.commons.lang.math.RandomUtils; ***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector; ***REMOVED*** ***REMOVED*** +import org.tmatesoft.svn.core.SVNException; @@ -191,12 +197,39 @@ public class RestPoolManager implements PoolManager { @QueryParam("category") String category, @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType, @DefaultValue("N") @QueryParam("skipJava") String skipJava, - @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException { + @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException { Algorithm algo= this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava); + ***REMOVED***publish algo if (publish){ service.addAlgToIs(algo); ***REMOVED*** + + ***REMOVED***update svn + for (Dependency d : algo.getDependencies()){ + + if (d.getType().equals("os")){ + List ls = new LinkedList(); + ls.add(d.getName()); + service.getOriginalContents("r_deb_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("cran")){ + List ls = new LinkedList(); + ls.add(d.getName()); + service.getOriginalContents("r_cran_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("custom")){ + List ls = new LinkedList(); + ls.add(d.getName()); + service.getOriginalContents("r_github_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("")){ + List ls = new LinkedList(); + ls.add(d.getName()); + service.getOriginalContents("r_deb_pkgs.txt", ls); + ***REMOVED*** + ***REMOVED*** + return service.addAlgorithmToVRE(algo, vre); ***REMOVED*** @@ -210,8 +243,38 @@ public class RestPoolManager implements PoolManager { @QueryParam("description") String description, @QueryParam("category") String category, @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType, - @DefaultValue("N") @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException { + @DefaultValue("N") @QueryParam("skipJava") String skipJava, + @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException { Algorithm algo= this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, skipJava); + ***REMOVED***publish algo + if (publish){ + service.addAlgToIs(algo); + ***REMOVED*** + + ***REMOVED***update svn + for (Dependency d : algo.getDependencies()){ + + if (d.getType().equals("os")){ + List ls = new LinkedList(); + ls.add(d.getName()); + service.getOriginalContents("r_deb_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("cran")){ + List ls = new LinkedList(); + ls.add(d.getName()); + service.getOriginalContents("r_cran_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("custom")){ + List ls = new LinkedList(); + ls.add(d.getName()); + service.getOriginalContents("r_github_pkgs.txt", ls); + ***REMOVED*** + if (d.getType().equals("")){ + List ls = new LinkedList(); + ls.add(d.getName()); + service.getOriginalContents("r_deb_pkgs.txt", ls); + ***REMOVED*** + ***REMOVED*** return service.addAlgorithmToHost(algo, hostname); ***REMOVED*** @@ -274,7 +337,7 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** - public static void main(String[] args) throws IOException, InterruptedException { + public static void main(String[] args) throws IOException, InterruptedException, SVNException { ***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); ***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); @@ -285,16 +348,27 @@ public class RestPoolManager implements PoolManager { ***REMOVED***System.out.println(aa.getAlgoFromIs()); - RestPoolManager a = new RestPoolManager(); + ***REMOVED***RestPoolManager a = new RestPoolManager(); ***REMOVED***a.addAlgorithmToVRE( ***REMOVED***"http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0", ***REMOVED***"/gcube/devNext/NextNext", ***REMOVED***null, null, "test", "transducerers", "N"); - a.addAlgorithmToHost( - "http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", - "dataminer1-devnext.d4science.org", - "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N"); +***REMOVED*** a.addAlgorithmToHost( +***REMOVED*** "http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", +***REMOVED*** "dataminer1-devnext.d4science.org", +***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N",false); +***REMOVED*** +***REMOVED*** + + PoolManager aa = new DataminerPoolManager(); + List ls = new LinkedList(); + String afa = "nunzio"; + String b = "galante"; + ls.add(b); + ls.add(afa); + + System.out.println(aa.getOriginalContents("r_deb_pkgs.txt",ls)); ***REMOVED*** @@ -344,6 +418,12 @@ public class RestPoolManager implements PoolManager { return null; ***REMOVED*** + @Override + public List getOriginalContents(String file, List ldep) throws SVNException { + ***REMOVED*** TODO Auto-generated method stub + return null; +***REMOVED*** + diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index a4fd7a3..fbe80ed 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -266,6 +266,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; ***REMOVED*** +import java.io.ByteArrayOutputStream; import java.io.File; ***REMOVED*** ***REMOVED*** @@ -274,6 +275,7 @@ import java.net.MalformedURLException; ***REMOVED*** import java.net.URLConnection; import java.net.UnknownHostException; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; ***REMOVED*** @@ -302,6 +304,12 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery; ***REMOVED*** ***REMOVED*** +import org.tmatesoft.svn.core.SVNException; +import org.tmatesoft.svn.core.SVNURL; +import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; +import org.tmatesoft.svn.core.io.SVNRepository; +import org.tmatesoft.svn.core.io.SVNRepositoryFactory; +import org.tmatesoft.svn.core.wc.SVNWCUtil; public class DataminerPoolManager implements PoolManager { @@ -549,6 +557,64 @@ public class DataminerPoolManager implements PoolManager { ***REMOVED*** + + + + private SVNRepository getSvnRepository(String url) throws SVNException { + String usr = "nunzioandrea.galante"; + String pwd = "Newuser1"; + SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); + ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager( + usr, pwd); + repository.setAuthenticationManager(authManager); + System.out.println(repository.getLocation()); + return repository; +***REMOVED*** + + + + @Override + public List getOriginalContents(String file, List ldep) throws SVNException { + final SVNRepository svnRepository = this.getSvnRepository("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); + try { + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + List aa = this.checkMatch(lines, ldep); + return aa; + + ***REMOVED*** finally { + svnRepository.closeSession(); + ***REMOVED*** +***REMOVED*** + + public List checkMatch(String[] lines, List ls) { + Set ss = new HashSet(ls); + ss.addAll(Arrays.asList(lines)); + + +***REMOVED*** for (int z = 0; z < aa.size(); z++) { +***REMOVED*** for (int i = 0; i < ls.size(); i++) { +***REMOVED*** if (!ls.get(i).equals(aa.get(z))) { +***REMOVED*** aa.add(ls.get(i)); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** + ***REMOVED******REMOVED*** + return new ArrayList<>(ss); +***REMOVED*** + + + public void checkSVNdeps(/*List listdep*/) throws IOException{ + ***REMOVED***take svn page (one for each kind of dep) + ***REMOVED***take deps from metadata + ***REMOVED***if dep.typology available in svn.typology nothing to do + ***REMOVED***else add dep to svn page +***REMOVED*** + + + + + public Algorithm extractAlgorithm(String url) throws IOException { return new AlgorithmPackageParser().parsePackage(url); ***REMOVED***