diff --git a/pom.xml b/pom.xml
index bc65c60..d36500c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -41,6 +41,11 @@
[4.0.0, 4.1.0)
+ org.tmatesoft.svnkit
+ svnkit
+ 1.8.5
+
+
commons-configuration
commons-configuration
1.10
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
index 1a20631..ffdd57e 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
@@ -70,15 +70,24 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
***REMOVED***
+import java.io.ByteArrayOutputStream;
+import java.io.File;
***REMOVED***
+import java.io.InputStream;
***REMOVED***
+import java.io.PrintStream;
import java.net.MalformedURLException;
***REMOVED***
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.LinkedHashMap;
***REMOVED***
***REMOVED***
import java.util.Map;
+import java.util.Scanner;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
@@ -87,11 +96,24 @@ import java.util.Vector;
import org.gcube.common.resources.gcore.ServiceEndpoint;
***REMOVED***
***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
+import org.tmatesoft.svn.core.SVNDepth;
+import org.tmatesoft.svn.core.SVNException;
+import org.tmatesoft.svn.core.SVNURL;
+import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
+import org.tmatesoft.svn.core.internal.io.dav.DAVRepositoryFactory;
+import org.tmatesoft.svn.core.io.SVNRepository;
+import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
+import org.tmatesoft.svn.core.wc.ISVNOptions;
+import org.tmatesoft.svn.core.wc.SVNClientManager;
+import org.tmatesoft.svn.core.wc.SVNRevision;
+import org.tmatesoft.svn.core.wc.SVNUpdateClient;
+import org.tmatesoft.svn.core.wc.SVNWCUtil;
***REMOVED***
@@ -168,6 +190,8 @@ public Cluster getClusterByHProxy() throws IOException {
***REMOVED***
+
+
/**
* Return the list of hosts (dataminers) in a given VRE
*
@@ -210,12 +234,25 @@ public Cluster getClusterByHProxy() throws IOException {
***REMOVED***
***REMOVED***
- public static void main(String[] args) throws IOException {
+ public static void main(String[] args) throws IOException, SVNException {
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
***REMOVED***System.out.println(a.getHProxy());
- System.out.println(a.getClusterByHProxy());
- System.out.println(a.listDataMinersByCluster());
+ ***REMOVED***System.out.println(a.getClusterByHProxy());
+ ***REMOVED***System.out.println(a.listDataMinersByCluster());
+
+
+ ***REMOVED***List list = new LinkedList();
+ ***REMOVED***Dependency aa = new Dependency();
+ ***REMOVED***aa.setName("testnunzio");
+ ***REMOVED***aa.setType("cran:");
+ ***REMOVED***list.add(aa);
+
+
+
+
+
+ ***REMOVED***a.checkSVNdep();
***REMOVED***System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
***REMOVED***System.out.println(a.listDataminersInVRE());
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
index 12643cf..d7fb0d1 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
@@ -31,9 +31,11 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest;
import java.net.MalformedURLException;
***REMOVED***
import java.net.UnknownHostException;
+***REMOVED***
import java.util.Set;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+import org.tmatesoft.svn.core.SVNException;
public interface PoolManager {
@@ -53,6 +55,7 @@ public interface PoolManager {
void addAlgToIs(Algorithm algo);
Set getAlgoFromIs();
+ List getOriginalContents(String file, List ldep) throws SVNException;
***REMOVED***
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
index d3eafe6..9b0cb10 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
@@ -142,11 +142,14 @@
***REMOVED******REMOVED***
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
+import java.awt.color.ICC_ColorSpace;
***REMOVED***
import java.net.MalformedURLException;
import java.net.ProxySelector;
***REMOVED***
import java.net.UnknownHostException;
+***REMOVED***
+***REMOVED***
import java.util.Random;
import java.util.Set;
import java.util.UUID;
@@ -159,11 +162,14 @@ import javax.ws.rs.QueryParam;
import org.apache.commons.lang.math.RandomUtils;
***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
***REMOVED***
***REMOVED***
+import org.tmatesoft.svn.core.SVNException;
@@ -191,12 +197,39 @@ public class RestPoolManager implements PoolManager {
@QueryParam("category") String category,
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
@DefaultValue("N") @QueryParam("skipJava") String skipJava,
- @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException {
+ @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException {
Algorithm algo= this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava);
+ ***REMOVED***publish algo
if (publish){
service.addAlgToIs(algo);
***REMOVED***
+
+ ***REMOVED***update svn
+ for (Dependency d : algo.getDependencies()){
+
+ if (d.getType().equals("os")){
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ service.getOriginalContents("r_deb_pkgs.txt", ls);
+ ***REMOVED***
+ if (d.getType().equals("cran")){
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ service.getOriginalContents("r_cran_pkgs.txt", ls);
+ ***REMOVED***
+ if (d.getType().equals("custom")){
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ service.getOriginalContents("r_github_pkgs.txt", ls);
+ ***REMOVED***
+ if (d.getType().equals("")){
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ service.getOriginalContents("r_deb_pkgs.txt", ls);
+ ***REMOVED***
+ ***REMOVED***
+
return service.addAlgorithmToVRE(algo, vre);
***REMOVED***
@@ -210,8 +243,38 @@ public class RestPoolManager implements PoolManager {
@QueryParam("description") String description,
@QueryParam("category") String category,
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
- @DefaultValue("N") @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
+ @DefaultValue("N") @QueryParam("skipJava") String skipJava,
+ @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException {
Algorithm algo= this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, skipJava);
+ ***REMOVED***publish algo
+ if (publish){
+ service.addAlgToIs(algo);
+ ***REMOVED***
+
+ ***REMOVED***update svn
+ for (Dependency d : algo.getDependencies()){
+
+ if (d.getType().equals("os")){
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ service.getOriginalContents("r_deb_pkgs.txt", ls);
+ ***REMOVED***
+ if (d.getType().equals("cran")){
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ service.getOriginalContents("r_cran_pkgs.txt", ls);
+ ***REMOVED***
+ if (d.getType().equals("custom")){
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ service.getOriginalContents("r_github_pkgs.txt", ls);
+ ***REMOVED***
+ if (d.getType().equals("")){
+ List ls = new LinkedList();
+ ls.add(d.getName());
+ service.getOriginalContents("r_deb_pkgs.txt", ls);
+ ***REMOVED***
+ ***REMOVED***
return service.addAlgorithmToHost(algo, hostname);
***REMOVED***
@@ -274,7 +337,7 @@ public class RestPoolManager implements PoolManager {
***REMOVED***
- public static void main(String[] args) throws IOException, InterruptedException {
+ public static void main(String[] args) throws IOException, InterruptedException, SVNException {
***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
@@ -285,16 +348,27 @@ public class RestPoolManager implements PoolManager {
***REMOVED***System.out.println(aa.getAlgoFromIs());
- RestPoolManager a = new RestPoolManager();
+ ***REMOVED***RestPoolManager a = new RestPoolManager();
***REMOVED***a.addAlgorithmToVRE(
***REMOVED***"http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0",
***REMOVED***"/gcube/devNext/NextNext",
***REMOVED***null, null, "test", "transducerers", "N");
- a.addAlgorithmToHost(
- "http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
- "dataminer1-devnext.d4science.org",
- "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N");
+***REMOVED*** a.addAlgorithmToHost(
+***REMOVED*** "http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
+***REMOVED*** "dataminer1-devnext.d4science.org",
+***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N",false);
+***REMOVED***
+***REMOVED***
+
+ PoolManager aa = new DataminerPoolManager();
+ List ls = new LinkedList();
+ String afa = "nunzio";
+ String b = "galante";
+ ls.add(b);
+ ls.add(afa);
+
+ System.out.println(aa.getOriginalContents("r_deb_pkgs.txt",ls));
***REMOVED***
@@ -344,6 +418,12 @@ public class RestPoolManager implements PoolManager {
return null;
***REMOVED***
+ @Override
+ public List getOriginalContents(String file, List ldep) throws SVNException {
+ ***REMOVED*** TODO Auto-generated method stub
+ return null;
+***REMOVED***
+
diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
index a4fd7a3..fbe80ed 100644
--- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
+++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
@@ -266,6 +266,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
***REMOVED***
+import java.io.ByteArrayOutputStream;
import java.io.File;
***REMOVED***
***REMOVED***
@@ -274,6 +275,7 @@ import java.net.MalformedURLException;
***REMOVED***
import java.net.URLConnection;
import java.net.UnknownHostException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
***REMOVED***
@@ -302,6 +304,12 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
***REMOVED***
***REMOVED***
+import org.tmatesoft.svn.core.SVNException;
+import org.tmatesoft.svn.core.SVNURL;
+import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
+import org.tmatesoft.svn.core.io.SVNRepository;
+import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
+import org.tmatesoft.svn.core.wc.SVNWCUtil;
public class DataminerPoolManager implements PoolManager {
@@ -549,6 +557,64 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED***
+
+
+
+ private SVNRepository getSvnRepository(String url) throws SVNException {
+ String usr = "nunzioandrea.galante";
+ String pwd = "Newuser1";
+ SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
+ ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(
+ usr, pwd);
+ repository.setAuthenticationManager(authManager);
+ System.out.println(repository.getLocation());
+ return repository;
+***REMOVED***
+
+
+
+ @Override
+ public List getOriginalContents(String file, List ldep) throws SVNException {
+ final SVNRepository svnRepository = this.getSvnRepository("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
+ try {
+ final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
+ String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
+ List aa = this.checkMatch(lines, ldep);
+ return aa;
+
+ ***REMOVED*** finally {
+ svnRepository.closeSession();
+ ***REMOVED***
+***REMOVED***
+
+ public List checkMatch(String[] lines, List ls) {
+ Set ss = new HashSet(ls);
+ ss.addAll(Arrays.asList(lines));
+
+
+***REMOVED*** for (int z = 0; z < aa.size(); z++) {
+***REMOVED*** for (int i = 0; i < ls.size(); i++) {
+***REMOVED*** if (!ls.get(i).equals(aa.get(z))) {
+***REMOVED*** aa.add(ls.get(i));
+***REMOVED*** ***REMOVED***
+***REMOVED*** ***REMOVED***
+ ***REMOVED******REMOVED***
+ return new ArrayList<>(ss);
+***REMOVED***
+
+
+ public void checkSVNdeps(/*List listdep*/) throws IOException{
+ ***REMOVED***take svn page (one for each kind of dep)
+ ***REMOVED***take deps from metadata
+ ***REMOVED***if dep.typology available in svn.typology nothing to do
+ ***REMOVED***else add dep to svn page
+***REMOVED***
+
+
+
+
+
public Algorithm extractAlgorithm(String url) throws IOException {
return new AlgorithmPackageParser().parsePackage(url);
***REMOVED***