gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146587 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-04-05 15:14:18 +00:00
parent 477932e99a
commit 481c022695
4 changed files with 114 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pom.xml
View File

@ -41,6 +41,11 @@
<version>[4.0.0, 4.1.0)</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.5</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>

43
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -70,15 +70,24 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
***REMOVED***
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
***REMOVED***
***REMOVED***
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.LinkedList;
***REMOVED***
import java.util.Map;
import java.util.Scanner;
***REMOVED***
import java.util.TreeMap;
import java.util.TreeSet;
@ -87,11 +96,24 @@ import java.util.Vector;
import org.gcube.common.resources.gcore.ServiceEndpoint;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.tmatesoft.svn.core.SVNDepth;
***REMOVED***
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.io.dav.DAVRepositoryFactory;
import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.wc.ISVNOptions;
import org.tmatesoft.svn.core.wc.SVNClientManager;
import org.tmatesoft.svn.core.wc.SVNRevision;
import org.tmatesoft.svn.core.wc.SVNUpdateClient;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
import au.com.bytecode.opencsv.CSVReader;
@ -168,6 +190,8 @@ public Cluster getClusterByHProxy() throws IOException ***REMOVED***
***REMOVED***
/**
* Return the list of hosts (dataminers) in a given VRE
*
@ -210,12 +234,25 @@ public Cluster getClusterByHProxy() throws IOException ***REMOVED***
***REMOVED***
***REMOVED***
public static void main(String[] args) throws IOException ***REMOVED***
public static void main(String[] args) throws IOException, SVNException ***REMOVED***
ISClient a = new ISClient();
ScopeProvider.instance.set("/gcube/devNext/NextNext");
***REMOVED***System.out.println(a.getHProxy());
System.out.println(a.getClusterByHProxy());
System.out.println(a.listDataMinersByCluster());
***REMOVED***System.out.println(a.getClusterByHProxy());
***REMOVED***System.out.println(a.listDataMinersByCluster());
***REMOVED***List<Dependency> list = new LinkedList<Dependency>();
***REMOVED***Dependency aa = new Dependency();
***REMOVED***aa.setName("testnunzio");
***REMOVED***aa.setType("cran:");
***REMOVED***list.add(aa);
***REMOVED***a.checkSVNdep();
***REMOVED***System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
***REMOVED***System.out.println(a.listDataminersInVRE());
***REMOVED***

3
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
View File

@ -27,12 +27,14 @@
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
public interface PoolManager ***REMOVED***
@ -53,6 +55,7 @@ public interface PoolManager ***REMOVED***
void addAlgToIs(Algorithm algo);
Set<Algorithm> getAlgoFromIs();
List<String> getOriginalContents(String file, List<String> ldep) throws SVNException;
***REMOVED***

66
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -266,6 +266,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
***REMOVED***
import java.io.InputStreamReader;
@ -274,6 +275,7 @@ import java.io.StringWriter;
***REMOVED***
import java.net.URLConnection;
***REMOVED***
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
***REMOVED***
@ -302,6 +304,12 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
***REMOVED***
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
public class DataminerPoolManager implements PoolManager ***REMOVED***
@ -549,6 +557,64 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
***REMOVED***
private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED***
String usr = "nunzioandrea.galante";
String pwd = "Newuser1";
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(
usr, pwd);
repository.setAuthenticationManager(authManager);
System.out.println(repository.getLocation());
return repository;
***REMOVED***
***REMOVED***
public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException ***REMOVED***
final SVNRepository svnRepository = this.getSvnRepository("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
try ***REMOVED***
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> aa = this.checkMatch(lines, ldep);
return aa;
***REMOVED*** finally ***REMOVED***
svnRepository.closeSession();
***REMOVED***
***REMOVED***
public List<String> checkMatch(String[] lines, List<String> ls) ***REMOVED***
Set<String> ss = new HashSet<String>(ls);
ss.addAll(Arrays.asList(lines));
***REMOVED*** for (int z = 0; z < aa.size(); z++) ***REMOVED***
***REMOVED*** for (int i = 0; i < ls.size(); i++) ***REMOVED***
***REMOVED*** if (!ls.get(i).equals(aa.get(z))) ***REMOVED***
***REMOVED*** aa.add(ls.get(i));
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
return new ArrayList<>(ss);
***REMOVED***
public void checkSVNdeps(/*List<Dependency> listdep*/) throws IOException***REMOVED***
***REMOVED***take svn page (one for each kind of dep)
***REMOVED***take deps from metadata
***REMOVED***if dep.typology available in svn.typology nothing to do
***REMOVED***else add dep to svn page
***REMOVED***
public Algorithm extractAlgorithm(String url) throws IOException ***REMOVED***
return new AlgorithmPackageParser().parsePackage(url);
***REMOVED***