git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146587 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
e57b57e59c
commit
5018f51090
5
pom.xml
5
pom.xml
|
|
@ -39,6 +39,11 @@
|
|||
<groupId>org.antlr</groupId>
|
||||
<artifactId>stringtemplate</artifactId>
|
||||
<version>[4.0.0, 4.1.0)</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.tmatesoft.svnkit</groupId>
|
||||
<artifactId>svnkit</artifactId>
|
||||
<version>1.8.5</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>commons-configuration</groupId>
|
||||
|
|
|
|||
|
|
@ -70,15 +70,24 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
|||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||
|
||||
***REMOVED***
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
***REMOVED***
|
||||
import java.io.InputStream;
|
||||
***REMOVED***
|
||||
import java.io.PrintStream;
|
||||
import java.net.MalformedURLException;
|
||||
***REMOVED***
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedHashMap;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import java.util.Map;
|
||||
import java.util.Scanner;
|
||||
import java.util.Set;
|
||||
import java.util.TreeMap;
|
||||
import java.util.TreeSet;
|
||||
|
|
@ -87,11 +96,24 @@ import java.util.Vector;
|
|||
import org.gcube.common.resources.gcore.ServiceEndpoint;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Domain;
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator;
|
||||
import org.gcube.resources.discovery.client.api.DiscoveryClient;
|
||||
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
||||
import org.tmatesoft.svn.core.SVNDepth;
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
import org.tmatesoft.svn.core.SVNURL;
|
||||
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
|
||||
import org.tmatesoft.svn.core.internal.io.dav.DAVRepositoryFactory;
|
||||
import org.tmatesoft.svn.core.io.SVNRepository;
|
||||
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
|
||||
import org.tmatesoft.svn.core.wc.ISVNOptions;
|
||||
import org.tmatesoft.svn.core.wc.SVNClientManager;
|
||||
import org.tmatesoft.svn.core.wc.SVNRevision;
|
||||
import org.tmatesoft.svn.core.wc.SVNUpdateClient;
|
||||
import org.tmatesoft.svn.core.wc.SVNWCUtil;
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
|
@ -168,6 +190,8 @@ public Cluster getClusterByHProxy() throws IOException {
|
|||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* Return the list of hosts (dataminers) in a given VRE
|
||||
*
|
||||
|
|
@ -210,12 +234,25 @@ public Cluster getClusterByHProxy() throws IOException {
|
|||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public static void main(String[] args) throws IOException {
|
||||
public static void main(String[] args) throws IOException, SVNException {
|
||||
ISClient a = new ISClient();
|
||||
ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
||||
***REMOVED***System.out.println(a.getHProxy());
|
||||
System.out.println(a.getClusterByHProxy());
|
||||
System.out.println(a.listDataMinersByCluster());
|
||||
***REMOVED***System.out.println(a.getClusterByHProxy());
|
||||
***REMOVED***System.out.println(a.listDataMinersByCluster());
|
||||
|
||||
|
||||
***REMOVED***List<Dependency> list = new LinkedList<Dependency>();
|
||||
***REMOVED***Dependency aa = new Dependency();
|
||||
***REMOVED***aa.setName("testnunzio");
|
||||
***REMOVED***aa.setType("cran:");
|
||||
***REMOVED***list.add(aa);
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
***REMOVED***a.checkSVNdep();
|
||||
***REMOVED***System.out.println(a.getDataminer("dataminer1-devnext.d4science.org").getDomain());
|
||||
***REMOVED***System.out.println(a.listDataminersInVRE());
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -31,9 +31,11 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest;
|
|||
import java.net.MalformedURLException;
|
||||
***REMOVED***
|
||||
import java.net.UnknownHostException;
|
||||
***REMOVED***
|
||||
import java.util.Set;
|
||||
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
|
||||
public interface PoolManager {
|
||||
|
||||
|
|
@ -53,6 +55,7 @@ public interface PoolManager {
|
|||
void addAlgToIs(Algorithm algo);
|
||||
|
||||
Set<Algorithm> getAlgoFromIs();
|
||||
List<String> getOriginalContents(String file, List<String> ldep) throws SVNException;
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -142,11 +142,14 @@
|
|||
***REMOVED******REMOVED***
|
||||
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
|
||||
|
||||
import java.awt.color.ICC_ColorSpace;
|
||||
***REMOVED***
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.ProxySelector;
|
||||
***REMOVED***
|
||||
import java.net.UnknownHostException;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
|
@ -159,11 +162,14 @@ import javax.ws.rs.QueryParam;
|
|||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ISClient;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySelector;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
|
||||
|
||||
|
||||
|
|
@ -191,12 +197,39 @@ public class RestPoolManager implements PoolManager {
|
|||
@QueryParam("category") String category,
|
||||
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
|
||||
@DefaultValue("N") @QueryParam("skipJava") String skipJava,
|
||||
@DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException {
|
||||
@DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException {
|
||||
Algorithm algo= this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava);
|
||||
|
||||
***REMOVED***publish algo
|
||||
if (publish){
|
||||
service.addAlgToIs(algo);
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***update svn
|
||||
for (Dependency d : algo.getDependencies()){
|
||||
|
||||
if (d.getType().equals("os")){
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
service.getOriginalContents("r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("cran")){
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
service.getOriginalContents("r_cran_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("custom")){
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
service.getOriginalContents("r_github_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("")){
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
service.getOriginalContents("r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
return service.addAlgorithmToVRE(algo, vre);
|
||||
***REMOVED***
|
||||
|
||||
|
|
@ -210,8 +243,38 @@ public class RestPoolManager implements PoolManager {
|
|||
@QueryParam("description") String description,
|
||||
@QueryParam("category") String category,
|
||||
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
|
||||
@DefaultValue("N") @QueryParam("skipJava") String skipJava) throws IOException, InterruptedException {
|
||||
@DefaultValue("N") @QueryParam("skipJava") String skipJava,
|
||||
@DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException {
|
||||
Algorithm algo= this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, skipJava);
|
||||
***REMOVED***publish algo
|
||||
if (publish){
|
||||
service.addAlgToIs(algo);
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***update svn
|
||||
for (Dependency d : algo.getDependencies()){
|
||||
|
||||
if (d.getType().equals("os")){
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
service.getOriginalContents("r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("cran")){
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
service.getOriginalContents("r_cran_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("custom")){
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
service.getOriginalContents("r_github_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("")){
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
service.getOriginalContents("r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return service.addAlgorithmToHost(algo, hostname);
|
||||
***REMOVED***
|
||||
|
||||
|
|
@ -274,7 +337,7 @@ public class RestPoolManager implements PoolManager {
|
|||
***REMOVED***
|
||||
|
||||
|
||||
public static void main(String[] args) throws IOException, InterruptedException {
|
||||
public static void main(String[] args) throws IOException, InterruptedException, SVNException {
|
||||
***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
|
||||
***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
|
||||
|
||||
|
|
@ -285,16 +348,27 @@ public class RestPoolManager implements PoolManager {
|
|||
***REMOVED***System.out.println(aa.getAlgoFromIs());
|
||||
|
||||
|
||||
RestPoolManager a = new RestPoolManager();
|
||||
***REMOVED***RestPoolManager a = new RestPoolManager();
|
||||
***REMOVED***a.addAlgorithmToVRE(
|
||||
***REMOVED***"http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0",
|
||||
***REMOVED***"/gcube/devNext/NextNext",
|
||||
***REMOVED***null, null, "test", "transducerers", "N");
|
||||
|
||||
a.addAlgorithmToHost(
|
||||
"http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
|
||||
"dataminer1-devnext.d4science.org",
|
||||
"ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N");
|
||||
***REMOVED*** a.addAlgorithmToHost(
|
||||
***REMOVED*** "http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
|
||||
***REMOVED*** "dataminer1-devnext.d4science.org",
|
||||
***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N",false);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
PoolManager aa = new DataminerPoolManager();
|
||||
List<String> ls = new LinkedList<String>();
|
||||
String afa = "nunzio";
|
||||
String b = "galante";
|
||||
ls.add(b);
|
||||
ls.add(afa);
|
||||
|
||||
System.out.println(aa.getOriginalContents("r_deb_pkgs.txt",ls));
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
|
@ -344,6 +418,12 @@ public class RestPoolManager implements PoolManager {
|
|||
return null;
|
||||
***REMOVED***
|
||||
|
||||
@Override
|
||||
public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException {
|
||||
***REMOVED*** TODO Auto-generated method stub
|
||||
return null;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -266,6 +266,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
|||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||
|
||||
***REMOVED***
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
|
@ -274,6 +275,7 @@ import java.net.MalformedURLException;
|
|||
***REMOVED***
|
||||
import java.net.URLConnection;
|
||||
import java.net.UnknownHostException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
***REMOVED***
|
||||
|
|
@ -302,6 +304,12 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
|||
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
import org.tmatesoft.svn.core.SVNURL;
|
||||
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
|
||||
import org.tmatesoft.svn.core.io.SVNRepository;
|
||||
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
|
||||
import org.tmatesoft.svn.core.wc.SVNWCUtil;
|
||||
|
||||
public class DataminerPoolManager implements PoolManager {
|
||||
|
||||
|
|
@ -549,6 +557,64 @@ public class DataminerPoolManager implements PoolManager {
|
|||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
private SVNRepository getSvnRepository(String url) throws SVNException {
|
||||
String usr = "nunzioandrea.galante";
|
||||
String pwd = "Newuser1";
|
||||
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
||||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(
|
||||
usr, pwd);
|
||||
repository.setAuthenticationManager(authManager);
|
||||
System.out.println(repository.getLocation());
|
||||
return repository;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException {
|
||||
final SVNRepository svnRepository = this.getSvnRepository("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
||||
try {
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
List<String> aa = this.checkMatch(lines, ldep);
|
||||
return aa;
|
||||
|
||||
***REMOVED*** finally {
|
||||
svnRepository.closeSession();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public List<String> checkMatch(String[] lines, List<String> ls) {
|
||||
Set<String> ss = new HashSet<String>(ls);
|
||||
ss.addAll(Arrays.asList(lines));
|
||||
|
||||
|
||||
***REMOVED*** for (int z = 0; z < aa.size(); z++) {
|
||||
***REMOVED*** for (int i = 0; i < ls.size(); i++) {
|
||||
***REMOVED*** if (!ls.get(i).equals(aa.get(z))) {
|
||||
***REMOVED*** aa.add(ls.get(i));
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
return new ArrayList<>(ss);
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public void checkSVNdeps(/*List<Dependency> listdep*/) throws IOException{
|
||||
***REMOVED***take svn page (one for each kind of dep)
|
||||
***REMOVED***take deps from metadata
|
||||
***REMOVED***if dep.typology available in svn.typology nothing to do
|
||||
***REMOVED***else add dep to svn page
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public Algorithm extractAlgorithm(String url) throws IOException {
|
||||
return new AlgorithmPackageParser().parsePackage(url);
|
||||
***REMOVED***
|
||||
|
|
|
|||
Loading…
Reference in New Issue