git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146587 82a268e6-3cf1-43bd-a215-b396298e98cf

pom.xml

@@ -39,6 +39,11 @@
 			<groupId>org.antlr</groupId>
 			<artifactId>stringtemplate</artifactId>
 			<version>[4.0.0, 4.1.0)</version>
+		</dependency>
+		<dependency>
+    <groupId>org.tmatesoft.svnkit</groupId>
+    <artifactId>svnkit</artifactId>
+    <version>1.8.5</version>
 </dependency>
 		<dependency>
 			<groupId>commons-configuration</groupId>

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java

@@ -70,15 +70,24 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
 import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
 
 ***REMOVED***
+import java.io.ByteArrayOutputStream;
+import java.io.File;
 ***REMOVED***
+import java.io.InputStream;
 ***REMOVED***
+import java.io.PrintStream;
 import java.net.MalformedURLException;
 ***REMOVED***
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 ***REMOVED***
 ***REMOVED***
 import java.util.Map;
+import java.util.Scanner;
 import java.util.Set;
 import java.util.TreeMap;
 import java.util.TreeSet;
@@ -87,11 +96,24 @@ import java.util.Vector;
 import org.gcube.common.resources.gcore.ServiceEndpoint;
 ***REMOVED***
 ***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
 import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
 ***REMOVED***
 import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
 import org.gcube.resources.discovery.client.api.DiscoveryClient;
 import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
+import org.tmatesoft.svn.core.SVNDepth;
+import org.tmatesoft.svn.core.SVNException;
+import org.tmatesoft.svn.core.SVNURL;
+import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
+import org.tmatesoft.svn.core.internal.io.dav.DAVRepositoryFactory;
+import org.tmatesoft.svn.core.io.SVNRepository;
+import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
+import org.tmatesoft.svn.core.wc.ISVNOptions;
+import org.tmatesoft.svn.core.wc.SVNClientManager;
+import org.tmatesoft.svn.core.wc.SVNRevision;
+import org.tmatesoft.svn.core.wc.SVNUpdateClient;
+import org.tmatesoft.svn.core.wc.SVNWCUtil;
 
 ***REMOVED***
 
@@ -168,6 +190,8 @@ public Cluster getClusterByHProxy() throws IOException {
 
 ***REMOVED***	
   
+	
+
   /**
    * Return the list of hosts (dataminers) in a given VRE
    * 
@@ -210,12 +234,25 @@ public Cluster getClusterByHProxy() throws IOException {
     ***REMOVED***
   ***REMOVED***
 
-  public static void main(String[] args) throws IOException {
+  public static void main(String[] args) throws IOException, SVNException {
 	ISClient a = new ISClient();
 	ScopeProvider.instance.set("/gcube/devNext/NextNext");
 	***REMOVED***System.out.println(a.getHProxy());
-	System.out.println(a.getClusterByHProxy());
+	***REMOVED***System.out.println(a.getClusterByHProxy());
-	System.out.println(a.listDataMinersByCluster());
+	***REMOVED***System.out.println(a.listDataMinersByCluster());
+	
+	
+	***REMOVED***List<Dependency> list = new LinkedList<Dependency>();
+	***REMOVED***Dependency aa = new Dependency();
+	***REMOVED***aa.setName("testnunzio");
+	***REMOVED***aa.setType("cran:");
+	***REMOVED***list.add(aa);
+	
+	
+	
+	
+	
+	***REMOVED***a.checkSVNdep();
 	***REMOVED***System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
 	***REMOVED***System.out.println(a.listDataminersInVRE());
 ***REMOVED***

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java

@@ -31,9 +31,11 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest;
 import java.net.MalformedURLException;
 ***REMOVED***
 import java.net.UnknownHostException;
+***REMOVED***
 import java.util.Set;
 
 import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+import org.tmatesoft.svn.core.SVNException;
 
 public interface PoolManager {
 
@@ -53,6 +55,7 @@ public interface PoolManager {
 	void addAlgToIs(Algorithm algo);
 	
 	Set<Algorithm> getAlgoFromIs();
+	List<String> getOriginalContents(String file, List<String> ldep) throws SVNException;
 	
 	
 ***REMOVED***

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java

@@ -142,11 +142,14 @@
 ***REMOVED******REMOVED***
 package org.gcube.dataanalysis.dataminer.poolmanager.rest;
 
+import java.awt.color.ICC_ColorSpace;
 ***REMOVED***
 import java.net.MalformedURLException;
 import java.net.ProxySelector;
 ***REMOVED***
 import java.net.UnknownHostException;
+***REMOVED***
+***REMOVED***
 import java.util.Random;
 import java.util.Set;
 import java.util.UUID;
@@ -159,11 +162,14 @@ import javax.ws.rs.QueryParam;
 
 import org.apache.commons.lang.math.RandomUtils;
 ***REMOVED***
+import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
 import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
+import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
 import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
 import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
 ***REMOVED***
 ***REMOVED***
+import org.tmatesoft.svn.core.SVNException;
 
 
 
@@ -191,12 +197,39 @@ public class RestPoolManager implements PoolManager {
       @QueryParam("category") String category,
       @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
       @DefaultValue("N") @QueryParam("skipJava") String skipJava,
-      @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException {
+      @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException {
     Algorithm algo= this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava);
     
+    ***REMOVED***publish algo
     if (publish){
         service.addAlgToIs(algo);
     ***REMOVED***
+    
+    ***REMOVED***update svn
+    for (Dependency d : algo.getDependencies()){
+    
+    if (d.getType().equals("os")){
+    	List<String> ls = new LinkedList<String>();
+        ls.add(d.getName());
+    	service.getOriginalContents("r_deb_pkgs.txt", ls);
+    ***REMOVED***
+    if (d.getType().equals("cran")){
+    	List<String> ls = new LinkedList<String>();
+        ls.add(d.getName());
+    	service.getOriginalContents("r_cran_pkgs.txt", ls);
+    ***REMOVED***
+    if (d.getType().equals("custom")){
+    	List<String> ls = new LinkedList<String>();
+        ls.add(d.getName());
+    	service.getOriginalContents("r_github_pkgs.txt", ls);
+    ***REMOVED***
+    if (d.getType().equals("")){
+    	List<String> ls = new LinkedList<String>();
+        ls.add(d.getName());
+    	service.getOriginalContents("r_deb_pkgs.txt", ls);
+    ***REMOVED***
+    ***REMOVED***
+    
     return service.addAlgorithmToVRE(algo, vre);
   ***REMOVED***
 
@@ -210,8 +243,38 @@ public class RestPoolManager implements PoolManager {
       @QueryParam("description") String description,
       @QueryParam("category") String category,
       @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
-      @DefaultValue("N") @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
+      @DefaultValue("N") @QueryParam("skipJava") String skipJava,
+      @DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException {
     Algorithm algo= this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, skipJava);
+    ***REMOVED***publish algo
+    if (publish){
+        service.addAlgToIs(algo);
+    ***REMOVED***
+    
+    ***REMOVED***update svn
+    for (Dependency d : algo.getDependencies()){
+    
+    if (d.getType().equals("os")){
+    	List<String> ls = new LinkedList<String>();
+        ls.add(d.getName());
+    	service.getOriginalContents("r_deb_pkgs.txt", ls);
+    ***REMOVED***
+    if (d.getType().equals("cran")){
+    	List<String> ls = new LinkedList<String>();
+        ls.add(d.getName());
+    	service.getOriginalContents("r_cran_pkgs.txt", ls);
+    ***REMOVED***
+    if (d.getType().equals("custom")){
+    	List<String> ls = new LinkedList<String>();
+        ls.add(d.getName());
+    	service.getOriginalContents("r_github_pkgs.txt", ls);
+    ***REMOVED***
+    if (d.getType().equals("")){
+    	List<String> ls = new LinkedList<String>();
+        ls.add(d.getName());
+    	service.getOriginalContents("r_deb_pkgs.txt", ls);
+    ***REMOVED***
+    ***REMOVED***
     return service.addAlgorithmToHost(algo, hostname);
   ***REMOVED***
 
@@ -274,7 +337,7 @@ public class RestPoolManager implements PoolManager {
 ***REMOVED***
 
 
-	public static void main(String[] args) throws IOException, InterruptedException {
+	public static void main(String[] args) throws IOException, InterruptedException, SVNException {
 		***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
 	    ***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
 		
@@ -285,16 +348,27 @@ public class RestPoolManager implements PoolManager {
 		***REMOVED***System.out.println(aa.getAlgoFromIs());
 
 		
-		RestPoolManager a = new RestPoolManager();
+		***REMOVED***RestPoolManager a = new RestPoolManager();
 		***REMOVED***a.addAlgorithmToVRE(
 		***REMOVED***"http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0",
 		***REMOVED***"/gcube/devNext/NextNext",
 		***REMOVED***null, null, "test", "transducerers", "N");
 				
-		a.addAlgorithmToHost(
+***REMOVED***		a.addAlgorithmToHost(
-		"http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
+***REMOVED***		"http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
-		"dataminer1-devnext.d4science.org",
+***REMOVED***		"dataminer1-devnext.d4science.org",
-		"ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N");
+***REMOVED***		"ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N",false);
+***REMOVED***	
+***REMOVED***		
+		
+		PoolManager aa = new DataminerPoolManager();
+		List<String> ls = new LinkedList<String>();
+		String afa = "nunzio";
+		String b = "galante";
+		ls.add(b);
+		ls.add(afa);
+		
+		System.out.println(aa.getOriginalContents("r_deb_pkgs.txt",ls));
 ***REMOVED***
 
 
@@ -344,6 +418,12 @@ public class RestPoolManager implements PoolManager {
 		return null;
 ***REMOVED***
 
+	@Override
+	public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException {
+		***REMOVED*** TODO Auto-generated method stub
+		return null;
+***REMOVED***
+
 	
 
 

src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java

@@ -266,6 +266,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
 import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
 
 ***REMOVED***
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 ***REMOVED***
 ***REMOVED***
@@ -274,6 +275,7 @@ import java.net.MalformedURLException;
 ***REMOVED***
 import java.net.URLConnection;
 import java.net.UnknownHostException;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashSet;
 ***REMOVED***
@@ -302,6 +304,12 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
 
 ***REMOVED***
 ***REMOVED***
+import org.tmatesoft.svn.core.SVNException;
+import org.tmatesoft.svn.core.SVNURL;
+import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
+import org.tmatesoft.svn.core.io.SVNRepository;
+import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
+import org.tmatesoft.svn.core.wc.SVNWCUtil;
 
 public class DataminerPoolManager implements PoolManager {
 
@@ -549,6 +557,64 @@ public class DataminerPoolManager implements PoolManager {
 
 ***REMOVED***
 
+	
+	
+	
+	private SVNRepository getSvnRepository(String url) throws SVNException {
+		String usr = "nunzioandrea.galante";
+		String pwd = "Newuser1";
+	    SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
+	    ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(
+	            usr, pwd);
+	    repository.setAuthenticationManager(authManager);
+	    System.out.println(repository.getLocation());
+	    return repository;
+***REMOVED***
+	
+
+	
+	@Override
+	public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException {
+	    final SVNRepository svnRepository = this.getSvnRepository("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
+	    try {
+	        final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+	        svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
+	        String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
+	        List<String> aa = this.checkMatch(lines, ldep);
+	        return aa;
+            
+	    ***REMOVED*** finally {
+	        svnRepository.closeSession();
+	    ***REMOVED***
+***REMOVED***
+	
+	public List<String> checkMatch(String[] lines, List<String> ls) {
+		Set<String> ss = new HashSet<String>(ls);
+		ss.addAll(Arrays.asList(lines));
+		
+		
+***REMOVED***		for (int z = 0; z < aa.size(); z++) {
+***REMOVED***			for (int i = 0; i < ls.size(); i++) {
+***REMOVED***				if (!ls.get(i).equals(aa.get(z))) {
+***REMOVED***					aa.add(ls.get(i));
+***REMOVED***			***REMOVED***
+***REMOVED***		***REMOVED***
+		***REMOVED******REMOVED***
+		return new ArrayList<>(ss);
+***REMOVED***
+	
+	
+	public void checkSVNdeps(/*List<Dependency> listdep*/) throws IOException{
+		***REMOVED***take svn page (one for each kind of dep)
+		***REMOVED***take deps from metadata
+		***REMOVED***if dep.typology available in svn.typology nothing to do
+		***REMOVED***else add dep to svn page	
+***REMOVED***	
+	
+	
+	
+	
+	
 	public Algorithm extractAlgorithm(String url) throws IOException {
 		return new AlgorithmPackageParser().parsePackage(url);
 ***REMOVED***