git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146601 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
481c022695
commit
2d55029f5b
|
|
@ -55,7 +55,7 @@ public interface PoolManager ***REMOVED***
|
|||
void addAlgToIs(Algorithm algo);
|
||||
|
||||
Set<Algorithm> getAlgoFromIs();
|
||||
List<String> getOriginalContents(String file, List<String> ldep) throws SVNException;
|
||||
List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException;
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -266,6 +266,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
|
|||
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
***REMOVED***
|
||||
|
|
@ -275,11 +276,14 @@ import java.io.StringWriter;
|
|||
***REMOVED***
|
||||
import java.net.URLConnection;
|
||||
***REMOVED***
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashSet;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import java.util.SortedSet;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.gcube.common.resources.gcore.GenericResource;
|
||||
|
|
@ -304,11 +308,16 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
|
|||
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.tmatesoft.svn.core.SVNCommitInfo;
|
||||
***REMOVED***
|
||||
import org.tmatesoft.svn.core.SVNURL;
|
||||
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
|
||||
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
|
||||
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
|
||||
import org.tmatesoft.svn.core.io.ISVNEditor;
|
||||
import org.tmatesoft.svn.core.io.SVNRepository;
|
||||
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
|
||||
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
|
||||
import org.tmatesoft.svn.core.wc.SVNWCUtil;
|
||||
|
||||
public class DataminerPoolManager implements PoolManager ***REMOVED***
|
||||
|
|
@ -531,7 +540,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
|
|||
|
||||
***REMOVED*** create the cluster (dataminers in the vre)
|
||||
Cluster cluster = new Cluster();
|
||||
***REMOVED***for (Host h : new ISClient().listDataMinersByCluster()) ***REMOVED***
|
||||
***REMOVED*** for (Host h : new ISClient().listDataMinersByCluster()) ***REMOVED***
|
||||
for (Host h : new ISClient().listDataminersInVRE()) ***REMOVED***
|
||||
cluster.addHost(h);
|
||||
***REMOVED***
|
||||
|
|
@ -557,64 +566,86 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
|
|||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED***
|
||||
String usr = "nunzioandrea.galante";
|
||||
String pwd = "Newuser1";
|
||||
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
||||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(
|
||||
usr, pwd);
|
||||
repository.setAuthenticationManager(authManager);
|
||||
System.out.println(repository.getLocation());
|
||||
return repository;
|
||||
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
||||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(usr, pwd);
|
||||
repository.setAuthenticationManager(authManager);
|
||||
System.out.println(repository.getLocation());
|
||||
return repository;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException ***REMOVED***
|
||||
final SVNRepository svnRepository = this.getSvnRepository("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
||||
try ***REMOVED***
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
List<String> aa = this.checkMatch(lines, ldep);
|
||||
return aa;
|
||||
|
||||
***REMOVED*** finally ***REMOVED***
|
||||
svnRepository.closeSession();
|
||||
***REMOVED***
|
||||
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException ***REMOVED***
|
||||
final SVNRepository svnRepository = this.getSvnRepository(
|
||||
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
||||
try ***REMOVED***
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
List<String> aa = this.checkMatch(lines, ldep);
|
||||
Collections.sort(aa);
|
||||
|
||||
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
||||
System.out.println("aaaa"+byteArrayOutputStream);
|
||||
|
||||
byte[] originalContents = byteArrayOutputStream.toByteArray();
|
||||
System.out.println(byteArrayOutputStream.toByteArray());
|
||||
|
||||
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
|
||||
commitEditor.openRoot(-1);
|
||||
commitEditor.openFile(file, -1);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
|
||||
for (String line : aa) ***REMOVED***
|
||||
baos.write(line.getBytes());
|
||||
baos.write("\n".getBytes());
|
||||
***REMOVED***
|
||||
System.out.println("bbbb"+baos);
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
commitEditor.applyTextDelta(file,md5(originalContents));
|
||||
|
||||
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
||||
new ByteArrayInputStream(bytes), commitEditor, true);
|
||||
commitEditor.closeFile(file, checksum);
|
||||
commitEditor.closeEdit();
|
||||
return aa;
|
||||
|
||||
***REMOVED*** finally ***REMOVED***
|
||||
svnRepository.closeSession();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public static String md5(byte[] contents) ***REMOVED***
|
||||
final byte[] tmp = new byte[1024];
|
||||
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
|
||||
try ***REMOVED***
|
||||
while (checksumStream.read(tmp) > 0) ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return checksumStream.getDigest();
|
||||
***REMOVED*** catch (IOException e) ***REMOVED***
|
||||
***REMOVED***never happens
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
***REMOVED*** finally ***REMOVED***
|
||||
SVNFileUtil.closeFile(checksumStream);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public List<String> checkMatch(String[] lines, List<String> ls) ***REMOVED***
|
||||
Set<String> ss = new HashSet<String>(ls);
|
||||
ss.addAll(Arrays.asList(lines));
|
||||
|
||||
|
||||
***REMOVED*** for (int z = 0; z < aa.size(); z++) ***REMOVED***
|
||||
***REMOVED*** for (int i = 0; i < ls.size(); i++) ***REMOVED***
|
||||
***REMOVED*** if (!ls.get(i).equals(aa.get(z))) ***REMOVED***
|
||||
***REMOVED*** aa.add(ls.get(i));
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
return new ArrayList<>(ss);
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public void checkSVNdeps(/*List<Dependency> listdep*/) throws IOException***REMOVED***
|
||||
***REMOVED***take svn page (one for each kind of dep)
|
||||
***REMOVED***take deps from metadata
|
||||
***REMOVED***if dep.typology available in svn.typology nothing to do
|
||||
***REMOVED***else add dep to svn page
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public Algorithm extractAlgorithm(String url) throws IOException ***REMOVED***
|
||||
return new AlgorithmPackageParser().parsePackage(url);
|
||||
***REMOVED***
|
||||
|
|
@ -673,11 +704,10 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
|
|||
***REMOVED***
|
||||
|
||||
private String getAlgoBody(Algorithm algo) ***REMOVED***
|
||||
return "<category>" + algo.getCategory() + "</category>" + "\n"
|
||||
+ "<clazz>" + algo.getClazz() + "</clazz>" + "\n" + "<algorithmType>" + algo.getAlgorithmType()
|
||||
+ "</algorithmType>" + "\n" + "<skipJava>" + algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>"
|
||||
+ algo.getPackageURL() + "</packageURL>" + "\n" + "<dependencies>"
|
||||
+ algo.getDependencies() + "</dependencies>";
|
||||
return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
|
||||
+ "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
|
||||
+ algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
|
||||
+ "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
|
||||
***REMOVED***
|
||||
|
||||
public void updateAlg(Algorithm algo) ***REMOVED***
|
||||
|
|
@ -703,7 +733,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
|
|||
private Algorithm convertAlgo(GenericResource a) ***REMOVED***
|
||||
Algorithm out = new Algorithm();
|
||||
|
||||
***REMOVED***out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
|
||||
***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
|
||||
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
|
||||
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
|
||||
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
|
||||
|
|
@ -722,14 +752,14 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
|
|||
return out;
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED*** public Algorithm getAlgoById(String id) ***REMOVED***
|
||||
***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) ***REMOVED***
|
||||
***REMOVED*** if (aa.getId().equals(id)) ***REMOVED***
|
||||
***REMOVED*** return aa;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
***REMOVED*** public Algorithm getAlgoById(String id) ***REMOVED***
|
||||
***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) ***REMOVED***
|
||||
***REMOVED*** if (aa.getId().equals(id)) ***REMOVED***
|
||||
***REMOVED*** return aa;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** return null;
|
||||
***REMOVED*** ***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
public Set<Algorithm> getAlgoFromIs() ***REMOVED***
|
||||
|
|
@ -737,7 +767,8 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
|
|||
|
||||
Set<Algorithm> out = new HashSet<Algorithm>();
|
||||
SimpleQuery query = queryFor(GenericResource.class);
|
||||
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'").setResult("$resource");
|
||||
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
|
||||
.setResult("$resource");
|
||||
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
|
||||
List<GenericResource> ds = client.submit(query);
|
||||
for (GenericResource a : ds) ***REMOVED***
|
||||
|
|
|
|||
Loading…
Reference in New Issue