gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@146601 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-04-06 08:59:46 +00:00
parent 5018f51090
commit 442b71c645
3 changed files with 250 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java
View File

@ -55,7 +55,7 @@ public interface PoolManager {
void addAlgToIs(Algorithm algo); void addAlgToIs(Algorithm algo);
Set<Algorithm> getAlgoFromIs(); Set<Algorithm> getAlgoFromIs();
List<String> getOriginalContents(String file, List<String> ldep) throws SVNException; List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException;
***REMOVED*** ***REMOVED***

341
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -171,9 +171,6 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.PropertiesBasedProxySel
***REMOVED*** ***REMOVED***
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
@Path("/") @Path("/")
public class RestPoolManager implements PoolManager { public class RestPoolManager implements PoolManager {
@ -182,141 +179,133 @@ public class RestPoolManager implements PoolManager {
private PoolManager service = new DataminerPoolManager(); private PoolManager service = new DataminerPoolManager();
/* /*
/scopes/<scope> POST ***REMOVED*** add an algorithm to all dataminers in the scope * /scopes/<scope> POST ***REMOVED*** add an algorithm to all dataminers in the scope
/hosts/<hostname> POST ***REMOVED*** add an algorithm to the given host * /hosts/<hostname> POST ***REMOVED*** add an algorithm to the given host
*/ */
@GET @GET
@Path("/scopes/add") @Path("/scopes/add")
@Produces("text/plain") @Produces("text/plain")
public String addAlgorithmToVRE( public String addAlgorithmToVRE(@QueryParam("algorithm") String algorithm, @QueryParam("vre") String vre,
@QueryParam("algorithm") String algorithm, @QueryParam("name") String name, @QueryParam("description") String description,
@QueryParam("vre") String vre, @QueryParam("category") String category,
@QueryParam("name") String name, @DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
@QueryParam("description") String description, @DefaultValue("N") @QueryParam("skipJava") String skipJava,
@QueryParam("category") String category, @DefaultValue("false") @QueryParam("publish") boolean publish)
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType, throws IOException, InterruptedException, SVNException {
@DefaultValue("N") @QueryParam("skipJava") String skipJava, Algorithm algo = this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava);
@DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException {
Algorithm algo= this.getAlgorithm(algorithm, vre, null, name, description, category, algorithmType, skipJava);
***REMOVED***publish algo
if (publish){
service.addAlgToIs(algo);
***REMOVED***
***REMOVED***update svn
for (Dependency d : algo.getDependencies()){
if (d.getType().equals("os")){
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.getOriginalContents("r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")){
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.getOriginalContents("r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("custom")){
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.getOriginalContents("r_github_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("")){
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.getOriginalContents("r_deb_pkgs.txt", ls);
***REMOVED***
***REMOVED***
return service.addAlgorithmToVRE(algo, vre);
***REMOVED***
@GET ***REMOVED*** publish algo
@Path("/hosts/add") if (publish) {
@Produces("text/plain") service.addAlgToIs(algo);
public String addAlgorithmToHost( ***REMOVED***
@QueryParam("algorithm") String algorithm,
@QueryParam("hostname") String hostname,
@QueryParam("name") String name,
@QueryParam("description") String description,
@QueryParam("category") String category,
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
@DefaultValue("N") @QueryParam("skipJava") String skipJava,
@DefaultValue("false") @QueryParam("publish") boolean publish) throws IOException, InterruptedException, SVNException {
Algorithm algo= this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType, skipJava);
***REMOVED***publish algo
if (publish){
service.addAlgToIs(algo);
***REMOVED***
***REMOVED***update svn
for (Dependency d : algo.getDependencies()){
if (d.getType().equals("os")){
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.getOriginalContents("r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")){
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.getOriginalContents("r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("custom")){
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.getOriginalContents("r_github_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("")){
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.getOriginalContents("r_deb_pkgs.txt", ls);
***REMOVED***
***REMOVED***
return service.addAlgorithmToHost(algo, hostname);
***REMOVED***
***REMOVED*** update svn
for (Dependency d : algo.getDependencies()) {
private Algorithm getAlgorithm( if (d.getType().equals("os")) {
String algorithm, List<String> ls = new LinkedList<String>();
String vre, ls.add(d.getName());
String hostname, service.updateSVN("r_deb_pkgs.txt", ls);
String name, ***REMOVED***
String description, if (d.getType().equals("cran")) {
String category, List<String> ls = new LinkedList<String>();
String algorithmType, ls.add(d.getName());
String skipJava) throws IOException, InterruptedException { service.updateSVN("r_cran_pkgs.txt", ls);
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre); ***REMOVED***
Algorithm algo = service.extractAlgorithm(algorithm); if (d.getType().equals("custom")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.updateSVN("r_github_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.updateSVN("r_deb_pkgs.txt", ls);
***REMOVED***
***REMOVED***
return service.addAlgorithmToVRE(algo, vre);
***REMOVED***
@GET
@Path("/hosts/add")
@Produces("text/plain")
public String addAlgorithmToHost(@QueryParam("algorithm") String algorithm, @QueryParam("hostname") String hostname,
@QueryParam("name") String name, @QueryParam("description") String description,
@QueryParam("category") String category,
@DefaultValue("transducerers") @QueryParam("algorithmType") String algorithmType,
@DefaultValue("N") @QueryParam("skipJava") String skipJava,
@DefaultValue("false") @QueryParam("publish") boolean publish)
throws IOException, InterruptedException, SVNException {
Algorithm algo = this.getAlgorithm(algorithm, null, hostname, name, description, category, algorithmType,
skipJava);
***REMOVED*** publish algo
if (publish) {
service.addAlgToIs(algo);
***REMOVED***
***REMOVED*** update svn
for (Dependency d : algo.getDependencies()) {
if (d.getType().equals("os")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.updateSVN("r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.updateSVN("r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("custom")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.updateSVN("r_github_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
service.updateSVN("r_deb_pkgs.txt", ls);
***REMOVED***
***REMOVED***
return service.addAlgorithmToHost(algo, hostname);
***REMOVED***
private Algorithm getAlgorithm(String algorithm, String vre, String hostname, String name, String description,
String category, String algorithmType, String skipJava) throws IOException, InterruptedException {
LOGGER.debug("Adding algorithm =" + algorithm + " to VRE =" + vre);
Algorithm algo = service.extractAlgorithm(algorithm);
if (algo.getCategory() == null) {
algo.setCategory(category);
***REMOVED*** else
algo.setCategory(algo.getCategory());
if (algo.getAlgorithmType() == null) {
algo.setAlgorithmType(algorithmType);
***REMOVED*** else
algo.setAlgorithmType(algo.getCategory());
if (algo.getSkipJava() == null) {
algo.setSkipJava(skipJava);
***REMOVED*** else
algo.setSkipJava(algo.getSkipJava());
if (algo.getName() == null) {
algo.setName(name);
***REMOVED*** else
algo.setName(algo.getName());
if (algo.getDescription() == null) {
algo.setDescription(description);
;
***REMOVED*** else
algo.setDescription(algo.getDescription());
return algo;
***REMOVED***
if (algo.getCategory() == null){
algo.setCategory(category);
***REMOVED*** else algo.setCategory(algo.getCategory());
if (algo.getAlgorithmType() == null){
algo.setAlgorithmType(algorithmType);
***REMOVED*** else algo.setAlgorithmType(algo.getCategory());
if (algo.getSkipJava() == null){
algo.setSkipJava(skipJava);
***REMOVED*** else algo.setSkipJava(algo.getSkipJava());
if (algo.getName() == null){
algo.setName(name);
***REMOVED*** else algo.setName(algo.getName());
if (algo.getDescription() == null){
algo.setDescription(description);;
***REMOVED*** else algo.setDescription(algo.getDescription());
return algo;
***REMOVED***
@GET @GET
@Path("/log") @Path("/log")
@Produces("text/plain") @Produces("text/plain")
@ -325,10 +314,6 @@ public class RestPoolManager implements PoolManager {
LOGGER.debug("Returning Log =" + logUrl); LOGGER.debug("Returning Log =" + logUrl);
return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl)); return service.getScriptFromURL(service.getURLfromWorkerLog(logUrl));
***REMOVED*** ***REMOVED***
@Override @Override
public Algorithm extractAlgorithm(String url) throws IOException { public Algorithm extractAlgorithm(String url) throws IOException {
@ -336,49 +321,44 @@ public class RestPoolManager implements PoolManager {
return null; return null;
***REMOVED*** ***REMOVED***
public static void main(String[] args) throws IOException, InterruptedException, SVNException { public static void main(String[] args) throws IOException, InterruptedException, SVNException {
***REMOVED***System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
***REMOVED***ProxySelector.setDefault(new PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); ***REMOVED*** ProxySelector.setDefault(new
***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
ScopeProvider.instance.set("/gcube/devNext/NextNext");
ScopeProvider.instance.set("/gcube/devNext/NextNext");
***REMOVED***PoolManager aa = new DataminerPoolManager();
***REMOVED***System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3")); ***REMOVED*** PoolManager aa = new DataminerPoolManager();
***REMOVED***System.out.println(aa.getAlgoFromIs()); ***REMOVED*** System.out.println(aa.getAlgoById("ICHTHYOP_MODEL_ONE_BY_ONE@3141d3aa-5f93-409f-b6f8-9fae0a6c0ee3"));
***REMOVED*** System.out.println(aa.getAlgoFromIs());
***REMOVED*** RestPoolManager a = new RestPoolManager();
***REMOVED*** a.addAlgorithmToVRE(
***REMOVED*** "http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0",
***REMOVED*** "/gcube/devNext/NextNext",
***REMOVED*** null, null, "test", "transducerers", "N");
***REMOVED*** a.addAlgorithmToHost(
***REMOVED*** "http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
***REMOVED*** "dataminer1-devnext.d4science.org",
***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers",
***REMOVED*** "N",false);
***REMOVED***
***REMOVED***
***REMOVED***RestPoolManager a = new RestPoolManager();
***REMOVED***a.addAlgorithmToVRE(
***REMOVED***"http:***REMOVED***data.d4science.org/S2h1RHZGd0JpWnBjZk9qTytQTndqcDRLVHNrQUt6QjhHbWJQNStIS0N6Yz0",
***REMOVED***"/gcube/devNext/NextNext",
***REMOVED***null, null, "test", "transducerers", "N");
***REMOVED*** a.addAlgorithmToHost(
***REMOVED*** "http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0",
***REMOVED*** "dataminer1-devnext.d4science.org",
***REMOVED*** "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", "N",false);
***REMOVED***
***REMOVED***
PoolManager aa = new DataminerPoolManager(); PoolManager aa = new DataminerPoolManager();
List<String> ls = new LinkedList<String>(); List<String> ls = new LinkedList<String>();
String afa = "nunzio"; String afa = "test";
String b = "galante";
ls.add(b);
ls.add(afa); ls.add(afa);
System.out.println(aa.getOriginalContents("r_deb_pkgs.txt",ls)); System.out.println(aa.updateSVN("r_deb_pkgs.txt", ls));
***REMOVED*** ***REMOVED***
@Override @Override
public void getLogId(Algorithm algo, String vre) { public void getLogId(Algorithm algo, String vre) {
***REMOVED*** TODO Auto-generated method stub ***REMOVED*** TODO Auto-generated method stub
***REMOVED***
***REMOVED***
@Override @Override
public String getScriptFromURL(URL logId) throws IOException { public String getScriptFromURL(URL logId) throws IOException {
@ -386,19 +366,17 @@ public class RestPoolManager implements PoolManager {
return null; return null;
***REMOVED*** ***REMOVED***
@Override @Override
public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException { public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException {
***REMOVED*** TODO Auto-generated method stub ***REMOVED*** TODO Auto-generated method stub
return null; return null;
***REMOVED*** ***REMOVED***
@Override @Override
public String addAlgorithmToHost(Algorithm algo, String hostname) public String addAlgorithmToHost(Algorithm algo, String hostname) throws IOException, InterruptedException {
throws IOException, InterruptedException { ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** TODO Auto-generated method stub return null;
return null; ***REMOVED***
***REMOVED***
@Override @Override
public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException {
@ -409,7 +387,7 @@ public class RestPoolManager implements PoolManager {
@Override @Override
public void addAlgToIs(Algorithm algo) { public void addAlgToIs(Algorithm algo) {
***REMOVED*** TODO Auto-generated method stub ***REMOVED*** TODO Auto-generated method stub
***REMOVED*** ***REMOVED***
@Override @Override
@ -418,15 +396,12 @@ public class RestPoolManager implements PoolManager {
return null; return null;
***REMOVED*** ***REMOVED***
@Override @Override
public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException { public List<String> updateSVN(String file, List<String> ldep) throws SVNException {
***REMOVED*** TODO Auto-generated method stub ***REMOVED*** TODO Auto-generated method stub
return null; return null;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***

151
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -266,6 +266,7 @@ import static org.gcube.resources.discovery.icclient.ICFactory.clientFor;
import static org.gcube.resources.discovery.icclient.ICFactory.queryFor; import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
***REMOVED*** ***REMOVED***
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
***REMOVED*** ***REMOVED***
@ -275,11 +276,14 @@ import java.net.MalformedURLException;
***REMOVED*** ***REMOVED***
import java.net.URLConnection; import java.net.URLConnection;
import java.net.UnknownHostException; import java.net.UnknownHostException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet; import java.util.HashSet;
***REMOVED*** ***REMOVED***
import java.util.Set; import java.util.Set;
import java.util.SortedSet;
import java.util.UUID; import java.util.UUID;
import org.gcube.common.resources.gcore.GenericResource; import org.gcube.common.resources.gcore.GenericResource;
@ -304,11 +308,16 @@ import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
import org.tmatesoft.svn.core.SVNCommitInfo;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
import org.tmatesoft.svn.core.io.ISVNEditor;
import org.tmatesoft.svn.core.io.SVNRepository; import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory; import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
import org.tmatesoft.svn.core.wc.SVNWCUtil; import org.tmatesoft.svn.core.wc.SVNWCUtil;
public class DataminerPoolManager implements PoolManager { public class DataminerPoolManager implements PoolManager {
@ -531,7 +540,7 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED*** create the cluster (dataminers in the vre) ***REMOVED*** create the cluster (dataminers in the vre)
Cluster cluster = new Cluster(); Cluster cluster = new Cluster();
***REMOVED***for (Host h : new ISClient().listDataMinersByCluster()) { ***REMOVED*** for (Host h : new ISClient().listDataMinersByCluster()) {
for (Host h : new ISClient().listDataminersInVRE()) { for (Host h : new ISClient().listDataminersInVRE()) {
cluster.addHost(h); cluster.addHost(h);
***REMOVED*** ***REMOVED***
@ -557,64 +566,86 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED*** ***REMOVED***
private SVNRepository getSvnRepository(String url) throws SVNException { private SVNRepository getSvnRepository(String url) throws SVNException {
String usr = "nunzioandrea.galante"; String usr = "nunzioandrea.galante";
String pwd = "Newuser1"; String pwd = "Newuser1";
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager( ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(usr, pwd);
usr, pwd); repository.setAuthenticationManager(authManager);
repository.setAuthenticationManager(authManager); System.out.println(repository.getLocation());
System.out.println(repository.getLocation()); return repository;
return repository; ***REMOVED***
@Override
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
final SVNRepository svnRepository = this.getSvnRepository(
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
try {
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> aa = this.checkMatch(lines, ldep);
Collections.sort(aa);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
System.out.println("aaaa"+byteArrayOutputStream);
byte[] originalContents = byteArrayOutputStream.toByteArray();
System.out.println(byteArrayOutputStream.toByteArray());
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : aa) {
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
System.out.println("bbbb"+baos);
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file,md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
commitEditor.closeEdit();
return aa;
***REMOVED*** finally {
svnRepository.closeSession();
***REMOVED***
***REMOVED*** ***REMOVED***
@Override
public List<String> getOriginalContents(String file, List<String> ldep) throws SVNException { public static String md5(byte[] contents) {
final SVNRepository svnRepository = this.getSvnRepository("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); final byte[] tmp = new byte[1024];
try { final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); try {
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); while (checksumStream.read(tmp) > 0) {
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); ***REMOVED***
List<String> aa = this.checkMatch(lines, ldep); ***REMOVED***
return aa; return checksumStream.getDigest();
***REMOVED*** catch (IOException e) {
***REMOVED*** finally { ***REMOVED***never happens
svnRepository.closeSession(); e.printStackTrace();
***REMOVED*** return null;
***REMOVED*** finally {
SVNFileUtil.closeFile(checksumStream);
***REMOVED***
***REMOVED*** ***REMOVED***
public List<String> checkMatch(String[] lines, List<String> ls) { public List<String> checkMatch(String[] lines, List<String> ls) {
Set<String> ss = new HashSet<String>(ls); Set<String> ss = new HashSet<String>(ls);
ss.addAll(Arrays.asList(lines)); ss.addAll(Arrays.asList(lines));
***REMOVED*** for (int z = 0; z < aa.size(); z++) {
***REMOVED*** for (int i = 0; i < ls.size(); i++) {
***REMOVED*** if (!ls.get(i).equals(aa.get(z))) {
***REMOVED*** aa.add(ls.get(i));
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
return new ArrayList<>(ss); return new ArrayList<>(ss);
***REMOVED*** ***REMOVED***
public void checkSVNdeps(/*List<Dependency> listdep*/) throws IOException{
***REMOVED***take svn page (one for each kind of dep)
***REMOVED***take deps from metadata
***REMOVED***if dep.typology available in svn.typology nothing to do
***REMOVED***else add dep to svn page
***REMOVED***
public Algorithm extractAlgorithm(String url) throws IOException { public Algorithm extractAlgorithm(String url) throws IOException {
return new AlgorithmPackageParser().parsePackage(url); return new AlgorithmPackageParser().parsePackage(url);
***REMOVED*** ***REMOVED***
@ -673,11 +704,10 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED*** ***REMOVED***
private String getAlgoBody(Algorithm algo) { private String getAlgoBody(Algorithm algo) {
return "<category>" + algo.getCategory() + "</category>" + "\n" return "<category>" + algo.getCategory() + "</category>" + "\n" + "<clazz>" + algo.getClazz() + "</clazz>"
+ "<clazz>" + algo.getClazz() + "</clazz>" + "\n" + "<algorithmType>" + algo.getAlgorithmType() + "\n" + "<algorithmType>" + algo.getAlgorithmType() + "</algorithmType>" + "\n" + "<skipJava>"
+ "</algorithmType>" + "\n" + "<skipJava>" + algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getSkipJava() + "</skipJava>" + "\n" + "<packageURL>" + algo.getPackageURL() + "</packageURL>"
+ algo.getPackageURL() + "</packageURL>" + "\n" + "<dependencies>" + "\n" + "<dependencies>" + algo.getDependencies() + "</dependencies>";
+ algo.getDependencies() + "</dependencies>";
***REMOVED*** ***REMOVED***
public void updateAlg(Algorithm algo) { public void updateAlg(Algorithm algo) {
@ -703,7 +733,7 @@ public class DataminerPoolManager implements PoolManager {
private Algorithm convertAlgo(GenericResource a) { private Algorithm convertAlgo(GenericResource a) {
Algorithm out = new Algorithm(); Algorithm out = new Algorithm();
***REMOVED***out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent()); ***REMOVED*** out.setId(a.profile().body().getElementsByTagName("id").item(0).getTextContent());
out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent()); out.setAlgorithmType(a.profile().body().getElementsByTagName("algorithmType").item(0).getTextContent());
out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent()); out.setCategory(a.profile().body().getElementsByTagName("category").item(0).getTextContent());
out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent()); out.setClazz(a.profile().body().getElementsByTagName("clazz").item(0).getTextContent());
@ -722,14 +752,14 @@ public class DataminerPoolManager implements PoolManager {
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** public Algorithm getAlgoById(String id) { ***REMOVED*** public Algorithm getAlgoById(String id) {
***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) { ***REMOVED*** for (Algorithm aa : this.getAlgoFromIs()) {
***REMOVED*** if (aa.getId().equals(id)) { ***REMOVED*** if (aa.getId().equals(id)) {
***REMOVED*** return aa; ***REMOVED*** return aa;
***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** return null; ***REMOVED*** return null;
***REMOVED******REMOVED*** ***REMOVED*** ***REMOVED***
@Override @Override
public Set<Algorithm> getAlgoFromIs() { public Set<Algorithm> getAlgoFromIs() {
@ -737,7 +767,8 @@ public class DataminerPoolManager implements PoolManager {
Set<Algorithm> out = new HashSet<Algorithm>(); Set<Algorithm> out = new HashSet<Algorithm>();
SimpleQuery query = queryFor(GenericResource.class); SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'").setResult("$resource"); query.addCondition("$resource/Profile/SecondaryType/text() eq 'StatisticalManagerAlgorithm'")
.setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class); DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query); List<GenericResource> ds = client.submit(query);
for (GenericResource a : ds) { for (GenericResource a : ds) {