gCubeSystem
/
dataminer-pool-manager
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146601 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-04-06 08:59:46 +00:00
parent 481c022695
commit 2d55029f5b
2 changed files with 92 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
View File

@ -55,7 +55,7 @@ public interface PoolManager ***REMOVED***
void addAlgToIs(Algorithm algo); void addAlgToIs(Algorithm algo);
Set<Algorithm> getAlgoFromIs(); Set<Algorithm> getAlgoFromIs();
List<String> getOriginalContents(String file, List<String> ldep) throws SVNException; List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException;
***REMOVED*** ***REMOVED***

151
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -266,6 +266,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.io.BufferedReader; import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
***REMOVED*** ***REMOVED***
@ -275,11 +276,14 @@ import java.io.StringWriter;
***REMOVED*** ***REMOVED***
import java.net.URLConnection; import java.net.URLConnection;
***REMOVED*** ***REMOVED***
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
import java.util.SortedSet;
import java.util.UUID; import java.util.UUID;
import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.GenericResource;
@ -304,11 +308,16 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
import org.tmatesoft.svn.core.SVNCommitInfo;
***REMOVED*** ***REMOVED***
import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
import org.tmatesoft.svn.core.io.ISVNEditor;
import org.tmatesoft.svn.core.io.SVNRepository; import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory; import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
import org.tmatesoft.svn.core.wc.SVNWCUtil; import org.tmatesoft.svn.core.wc.SVNWCUtil;
public class DataminerPoolManager implements PoolManager ***REMOVED*** public class DataminerPoolManager implements PoolManager ***REMOVED***
@ -531,7 +540,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
***REMOVED*** create the cluster (dataminers in the vre) ***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster(); Cluster cluster = new Cluster();
***REMOVED***for (Host h : new ISClient().listDataMinersByCluster()) ***REMOVED*** ***REMOVED*** for (Host h : new ISClient().listDataMinersByCluster()) ***REMOVED***
for (Host h : new ISClient().listDataminersInVRE()) ***REMOVED*** for (Host h : new ISClient().listDataminersInVRE()) ***REMOVED***
cluster.addHost(h); cluster.addHost(h);
***REMOVED*** ***REMOVED***
@ -557,64 +566,86 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
***REMOVED*** ***REMOVED***
private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED*** private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED***
String usr = "nunzioandrea.galante"; String usr = "nunzioandrea.galante";
String pwd = "Newuser1"; String pwd = "Newuser1";
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager( ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(usr, pwd);
usr, pwd); repository.setAuthenticationManager(authManager);
repository.setAuthenticationManager(authManager); System.out.println(repository.getLocation());
System.out.println(repository.getLocation()); return repository;
return repository;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException ***REMOVED*** public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException ***REMOVED***
final SVNRepository svnRepository = this.getSvnRepository("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); final SVNRepository svnRepository = this.getSvnRepository(
try ***REMOVED*** "https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); try ***REMOVED***
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
List<String> aa = this.checkMatch(lines, ldep); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
return aa; List<String> aa = this.checkMatch(lines, ldep);
Collections.sort(aa);
***REMOVED*** finally ***REMOVED***
svnRepository.closeSession(); final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
***REMOVED*** System.out.println("aaaa"+byteArrayOutputStream);
byte[] originalContents = byteArrayOutputStream.toByteArray();
System.out.println(byteArrayOutputStream.toByteArray());
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : aa) ***REMOVED***
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
System.out.println("bbbb"+baos);
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file,md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
commitEditor.closeEdit();
return aa;
***REMOVED*** finally ***REMOVED***
svnRepository.closeSession();
***REMOVED***
***REMOVED***
public static String md5(byte[] contents) ***REMOVED***
final byte[] tmp = new byte[1024];
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
try ***REMOVED***
while (checksumStream.read(tmp) > 0) ***REMOVED***
***REMOVED***
***REMOVED***
return checksumStream.getDigest();
***REMOVED*** catch (IOException e) ***REMOVED***
***REMOVED***never happens
e.printStackTrace();
return null;
***REMOVED*** finally ***REMOVED***
SVNFileUtil.closeFile(checksumStream);
***REMOVED***
***REMOVED*** ***REMOVED***
public List<String> checkMatch(String[] lines, List<String> ls) ***REMOVED*** public List<String> checkMatch(String[] lines, List<String> ls) ***REMOVED***
Set<String> ss = new HashSet<String>(ls); Set<String> ss = new HashSet<String>(ls);
ss.addAll(Arrays.asList(lines)); ss.addAll(Arrays.asList(lines));
***REMOVED*** for (int z = 0; z < aa.size(); z++) ***REMOVED***
***REMOVED*** for (int i = 0; i < ls.size(); i++) ***REMOVED***
***REMOVED*** if (!ls.get(i).equals(aa.get(z))) ***REMOVED***
***REMOVED*** aa.add(ls.get(i));
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
return new ArrayList<>(ss); return new ArrayList<>(ss);
***REMOVED*** ***REMOVED***
public void checkSVNdeps(/*List<Dependency> listdep*/) throws IOException***REMOVED***
***REMOVED***take svn page (one for each kind of dep)
***REMOVED***take deps from metadata
***REMOVED***if dep.typology available in svn.typology nothing to do
***REMOVED***else add dep to svn page
***REMOVED***
public Algorithm extractAlgorithm(String url) throws IOException ***REMOVED*** public Algorithm extractAlgorithm(String url) throws IOException ***REMOVED***
return new AlgorithmPackageParser().parsePackage(url); return new AlgorithmPackageParser().parsePackage(url);
***REMOVED*** ***REMOVED***
@ -673,11 +704,10 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
***REMOVED*** ***REMOVED***
private String getAlgoBody(Algorithm algo) ***REMOVED*** private String getAlgoBody(Algorithm algo) ***REMOVED***
return "<category>" + algo.getCategory() + "</category>" + "\n" return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
+ "<clazz>" + algo.getClazz() + "</clazz>" + "\n" + "<algorithmType>" + algo.getAlgorithmType() + "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
+ "</algorithmType>" + "\n" + "<skipJava>" + algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
+ algo.getPackageURL() + "</packageURL>" + "\n" + "<dependencies>" + "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
+ algo.getDependencies() + "</dependencies>";
***REMOVED*** ***REMOVED***
public void updateAlg(Algorithm algo) ***REMOVED*** public void updateAlg(Algorithm algo) ***REMOVED***
@ -703,7 +733,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
private Algorithm convertAlgo(GenericResource a) ***REMOVED*** private Algorithm convertAlgo(GenericResource a) ***REMOVED***
Algorithm out = new Algorithm(); Algorithm out = new Algorithm();
***REMOVED***out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); ***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
@ -722,14 +752,14 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
return out; return out;
***REMOVED*** ***REMOVED***
***REMOVED*** public Algorithm getAlgoById(String id) ***REMOVED*** ***REMOVED*** public Algorithm getAlgoById(String id) ***REMOVED***
***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) ***REMOVED*** ***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) ***REMOVED***
***REMOVED*** if (aa.getId().equals(id)) ***REMOVED*** ***REMOVED*** if (aa.getId().equals(id)) ***REMOVED***
***REMOVED*** return aa; ***REMOVED*** return aa;
***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED******REMOVED*** ***REMOVED*** return null;
***REMOVED******REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
public Set<Algorithm> getAlgoFromIs() ***REMOVED*** public Set<Algorithm> getAlgoFromIs() ***REMOVED***
@ -737,7 +767,8 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
Set<Algorithm> out = new HashSet<Algorithm>(); Set<Algorithm> out = new HashSet<Algorithm>();
SimpleQuery query = queryFor(GenericResource.class); SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'").setResult("$resource"); query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
.setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class); DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query); List<GenericResource> ds = client.submit(query);
for (GenericResource a : ds) ***REMOVED*** for (GenericResource a : ds) ***REMOVED***