From c381af3416bd3d5a86dcc66627b17f87aebb335c Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Sat, 22 Apr 2017 17:24:33 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@147118 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../poolmanager/ansible/AnsibleWorker.java | 136 +++++++++++++++++- .../ansiblebridge/AnsibleBridge.java | 9 +- .../poolmanager/rest/PoolManager.java | 4 +- .../poolmanager/rest/RestPoolManager.java | 117 ++++++++------- .../service/DataminerPoolManager.java | 109 ++++---------- src/test/java/DataminerPoolManagerTest.java | 2 +- .../DataminerPoolManagerTest.java | 2 +- 7 files changed, 236 insertions(+), 143 deletions(-) diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index 6465973..573f2c4 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -1,17 +1,39 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansible; import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedList; +import java.util.List; import java.util.Scanner; +import java.util.Set; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Inventory; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Playbook; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.AlgorithmSet; +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; +import org.tmatesoft.svn.core.SVNException; +import org.tmatesoft.svn.core.SVNURL; +import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; +import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; +import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream; +import org.tmatesoft.svn.core.io.ISVNEditor; +import org.tmatesoft.svn.core.io.SVNRepository; +import org.tmatesoft.svn.core.io.SVNRepositoryFactory; +import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator; +import org.tmatesoft.svn.core.wc.SVNWCUtil; /** * This class is responsible for the interface with ansible, retrieving log, @@ -96,7 +118,7 @@ public class AnsibleWorker { - public void apply(PrintStream ps) throws IOException { + public void apply(AlgorithmSet as, PrintStream ps, boolean updateSVN) throws IOException, InterruptedException, SVNException { // TODO execute the playbook and return output System.out.println(this.getWorkdir()); try { @@ -105,6 +127,41 @@ public class AnsibleWorker { inheritIO(p.getInputStream(), ps); inheritIO(p.getErrorStream(), ps); + if (updateSVN){ + int exitValue = p.waitFor(); + if (exitValue == 0){ + + for (Algorithm algo : as.getAlgorithms()){ + + for (Dependency d : algo.getDependencies()) { + + if (d.getType().equals("os")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN("r_deb_pkgs.txt", ls); + } + if (d.getType().equals("cran")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN("r_cran_pkgs.txt", ls); + } + if (d.getType().equals("github")) { + List ls = new LinkedList(); + ls.add(d.getName()); + this.updateSVN("r_github_pkgs.txt", ls); + } + } + } + + + } + + + + + + } + } catch (IOException e) { e.printStackTrace(); } @@ -113,6 +170,83 @@ public class AnsibleWorker { + private SVNRepository getSvnRepository(String url) throws SVNException { + SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); + ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); + repository.setAuthenticationManager(authManager); + System.out.println(repository.getLocation()); + + return repository; + } + + + + public List updateSVN(String file, List ldep) throws SVNException, IOException { + final SVNRepository svnRepository = this.getSvnRepository( + "https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); + try { + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + List aa = this.checkMatch(lines, ldep); + Collections.sort(aa); + + final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + + byte[] originalContents = byteArrayOutputStream.toByteArray(); + + final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); + commitEditor.openRoot(-1); + commitEditor.openFile(file, -1); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + for (String line : aa) { + baos.write(line.getBytes()); + baos.write("\n".getBytes()); + } + byte[] bytes = baos.toByteArray(); + + commitEditor.applyTextDelta(file,md5(originalContents)); + + final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, + new ByteArrayInputStream(bytes), commitEditor, true); + commitEditor.closeFile(file, checksum); + commitEditor.closeEdit(); + return aa; + + } finally { + svnRepository.closeSession(); + } + } + + + + public static String md5(byte[] contents) { + final byte[] tmp = new byte[1024]; + final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5"); + try { + while (checksumStream.read(tmp) > 0) { + // + } + return checksumStream.getDigest(); + } catch (IOException e) { + //never happens + e.printStackTrace(); + return null; + } finally { + SVNFileUtil.closeFile(checksumStream); + } +} + + public List checkMatch(String[] lines, List ls) { + Set ss = new HashSet(ls); + ss.addAll(Arrays.asList(lines)); + return new ArrayList<>(ss); + } + + + private static void inheritIO(final InputStream src, final PrintStream dest) { new Thread(new Runnable() { public void run() { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index b74dba1..32fa9fe 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -32,6 +32,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.comparator.HostComparator; import org.slf4j.LoggerFactory; +import org.tmatesoft.svn.core.SVNException; public class AnsibleBridge { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); @@ -145,13 +146,13 @@ public class AnsibleBridge { } } - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster) throws IOException { + public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster, boolean updateSVN) throws IOException, InterruptedException, SVNException { - return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString()); + return applyAlgorithmSetToCluster (as,cluster,UUID.randomUUID().toString(),updateSVN); } - public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid) throws IOException { + public AnsibleWorker applyAlgorithmSetToCluster(AlgorithmSet as, Cluster cluster,String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException { AnsibleWorker worker = new AnsibleWorker(new File(this.getWorkDir(), uuid)); @@ -207,7 +208,7 @@ public class AnsibleBridge { //System.setErr(console); - worker.apply(ps); + worker.apply(as,ps,updateSVN); //System.setOut(console); //worker.apply(); System.out.println("Log stored to to " + n.getAbsolutePath()); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java index e182140..4f87fb0 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/PoolManager.java @@ -39,8 +39,8 @@ import org.tmatesoft.svn.core.SVNException; public interface PoolManager { - String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException; - String addAlgorithmToHost(Algorithm algo, String host) throws IOException, InterruptedException; + String addAlgorithmToVRE(Algorithm algo, String vre, boolean svn ) throws IOException, InterruptedException; + String addAlgorithmToHost(Algorithm algo, String host, boolean svn) throws IOException, InterruptedException; Algorithm extractAlgorithm(String url) throws IOException; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index d1e615c..b5e50a0 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -203,28 +203,28 @@ public class RestPoolManager implements PoolManager { service.addAlgToIs(algo); } // update svn - if (updateSVN){ - for (Dependency d : algo.getDependencies()) { +// if (updateSVN){ +// for (Dependency d : algo.getDependencies()) { +// +// if (d.getType().equals("os")) { +// List ls = new LinkedList(); +// ls.add(d.getName()); +// service.updateSVN("r_deb_pkgs.txt", ls); +// } +// if (d.getType().equals("cran")) { +// List ls = new LinkedList(); +// ls.add(d.getName()); +// service.updateSVN("r_cran_pkgs.txt", ls); +// } +// if (d.getType().equals("github")) { +// List ls = new LinkedList(); +// ls.add(d.getName()); +// service.updateSVN("r_github_pkgs.txt", ls); +// } +// } + //} - if (d.getType().equals("os")) { - List ls = new LinkedList(); - ls.add(d.getName()); - service.updateSVN("r_deb_pkgs.txt", ls); - } - if (d.getType().equals("cran")) { - List ls = new LinkedList(); - ls.add(d.getName()); - service.updateSVN("r_cran_pkgs.txt", ls); - } - if (d.getType().equals("github")) { - List ls = new LinkedList(); - ls.add(d.getName()); - service.updateSVN("r_github_pkgs.txt", ls); - } - } - } - - return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get()); + return service.addAlgorithmToVRE(algo, ScopeProvider.instance.get(),updateSVN); } @@ -252,27 +252,27 @@ public class RestPoolManager implements PoolManager { } // update svn - if (updateSVN){ - for (Dependency d : algo.getDependencies()) { - - if (d.getType().equals("os")) { - List ls = new LinkedList(); - ls.add(d.getName()); - service.updateSVN("r_deb_pkgs.txt", ls); - } - if (d.getType().equals("cran")) { - List ls = new LinkedList(); - ls.add(d.getName()); - service.updateSVN("r_cran_pkgs.txt", ls); - } - if (d.getType().equals("github")) { - List ls = new LinkedList(); - ls.add(d.getName()); - service.updateSVN("r_github_pkgs.txt", ls); - } - } - } - return service.addAlgorithmToHost(algo, hostname); +// if (updateSVN){ +// for (Dependency d : algo.getDependencies()) { +// +// if (d.getType().equals("os")) { +// List ls = new LinkedList(); +// ls.add(d.getName()); +// service.updateSVN("r_deb_pkgs.txt", ls); +// } +// if (d.getType().equals("cran")) { +// List ls = new LinkedList(); +// ls.add(d.getName()); +// service.updateSVN("r_cran_pkgs.txt", ls); +// } +// if (d.getType().equals("github")) { +// List ls = new LinkedList(); +// ls.add(d.getName()); +// service.updateSVN("r_github_pkgs.txt", ls); +// } +// } + //} + return service.addAlgorithmToHost(algo, hostname,updateSVN); } private Algorithm getAlgorithm(String algorithm, String vre, String hostname, String name, String description, @@ -348,10 +348,10 @@ public class RestPoolManager implements PoolManager { // false); a.addAlgorithmToHost( - "http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", - "dataminer2-d-d4s.d4science.org", + "http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", + "dataminer1-devnext.d4science.org", "ICHTHYOP_MODEL_ONE_BY_ONE", null, "ICHTHYOP_MODEL", "transducerers", - "N",false, false); + "N",false, true); @@ -375,17 +375,7 @@ public class RestPoolManager implements PoolManager { return null; } - @Override - public String addAlgorithmToVRE(Algorithm algo, String vre) throws IOException, InterruptedException { - // TODO Auto-generated method stub - return null; - } - @Override - public String addAlgorithmToHost(Algorithm algo, String hostname) throws IOException, InterruptedException { - // TODO Auto-generated method stub - return null; - } @Override public URL getURLfromWorkerLog(String logUrl) throws MalformedURLException, UnknownHostException { @@ -411,6 +401,23 @@ public class RestPoolManager implements PoolManager { // TODO Auto-generated method stub return null; } + + + + @Override + public String addAlgorithmToVRE(Algorithm algo, String vre, boolean svn) throws IOException, InterruptedException { + // TODO Auto-generated method stub + return null; + } + + + + @Override + public String addAlgorithmToHost(Algorithm algo, String host, boolean svn) + throws IOException, InterruptedException { + // TODO Auto-generated method stub + return null; + } } diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 426b1d3..844d32f 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -476,7 +476,7 @@ public class DataminerPoolManager implements PoolManager { * @param vre * */ - public String addAlgorithmToVRE(Algorithm algorithm, final String vre) throws IOException { + public String addAlgorithmToVRE(Algorithm algorithm, final String vre, final boolean updateSVN) throws IOException { // create a fake algorithm set final AlgorithmSet algoSet = new AlgorithmSet(); algoSet.setName("fake"); @@ -488,10 +488,18 @@ public class DataminerPoolManager implements PoolManager { public void run() { // TODO Auto-generated method stub try { - addAlgorithmsToVRE(algoSet, vre, uuid); + try { + addAlgorithmsToVRE(algoSet, vre, uuid, updateSVN); + } catch (SVNException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } } }).start(); @@ -501,7 +509,7 @@ public class DataminerPoolManager implements PoolManager { return uuid; } - public String addAlgorithmToHost(Algorithm algorithm, final String hostname) throws IOException { + public String addAlgorithmToHost(Algorithm algorithm, final String hostname, final boolean updateSVN) throws IOException { // create a fake algorithm set final AlgorithmSet algoSet = new AlgorithmSet(); algoSet.setName("fake"); @@ -513,10 +521,16 @@ public class DataminerPoolManager implements PoolManager { public void run() { // TODO Auto-generated method stub try { - addAlgorithmsToHost(algoSet, hostname, uuid); + addAlgorithmsToHost(algoSet, hostname, uuid, updateSVN); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); + } catch (InterruptedException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } catch (SVNException e) { + // TODO Auto-generated catch block + e.printStackTrace(); } } }).start(); @@ -537,7 +551,7 @@ public class DataminerPoolManager implements PoolManager { return new File(n.getPath()).toURI().toURL(); } - public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid) throws IOException { + public String addAlgorithmsToVRE(AlgorithmSet algorithms, String vre, String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException { // create the cluster (dataminers in the vre) Cluster cluster = new Cluster(); @@ -548,11 +562,11 @@ public class DataminerPoolManager implements PoolManager { // apply the changes AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid).getWorkerId(); + return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid, updateSVN).getWorkerId(); } - public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid) throws IOException { + public String addAlgorithmsToHost(AlgorithmSet algorithms, String hostname, String uuid, boolean updateSVN) throws IOException, InterruptedException, SVNException { // create the cluster (dataminers in the vre) Cluster cluster = new Cluster(); @@ -563,7 +577,7 @@ public class DataminerPoolManager implements PoolManager { // apply the changes AnsibleBridge a = new AnsibleBridge(); - return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid).getWorkerId(); + return a.applyAlgorithmSetToCluster(algorithms, cluster, uuid,updateSVN ).getWorkerId(); } @@ -571,81 +585,10 @@ public class DataminerPoolManager implements PoolManager { - private SVNRepository getSvnRepository(String url) throws SVNException { - SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); - ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); - repository.setAuthenticationManager(authManager); - System.out.println(repository.getLocation()); - return repository; - } - @Override - public List updateSVN(String file, List ldep) throws SVNException, IOException { - final SVNRepository svnRepository = this.getSvnRepository( - "https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); - try { - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - List aa = this.checkMatch(lines, ldep); - Collections.sort(aa); - - final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); - - byte[] originalContents = byteArrayOutputStream.toByteArray(); - - final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); - commitEditor.openRoot(-1); - commitEditor.openFile(file, -1); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - for (String line : aa) { - baos.write(line.getBytes()); - baos.write("\n".getBytes()); - } - byte[] bytes = baos.toByteArray(); - - commitEditor.applyTextDelta(file,md5(originalContents)); - - final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, - new ByteArrayInputStream(bytes), commitEditor, true); - commitEditor.closeFile(file, checksum); - commitEditor.closeEdit(); - return aa; - - } finally { - svnRepository.closeSession(); - } - } - - public static String md5(byte[] contents) { - final byte[] tmp = new byte[1024]; - final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5"); - try { - while (checksumStream.read(tmp) > 0) { - // - } - return checksumStream.getDigest(); - } catch (IOException e) { - //never happens - e.printStackTrace(); - return null; - } finally { - SVNFileUtil.closeFile(checksumStream); - } -} - - public List checkMatch(String[] lines, List ls) { - Set ss = new HashSet(ls); - ss.addAll(Arrays.asList(lines)); - return new ArrayList<>(ss); - } - - public Algorithm extractAlgorithm(String url) throws IOException { return new AlgorithmPackageParser().parsePackage(url); @@ -778,4 +721,12 @@ public class DataminerPoolManager implements PoolManager { return out; } + + + @Override + public List updateSVN(String file, List ldep) throws SVNException, IOException { + // TODO Auto-generated method stub + return null; + } + } diff --git a/src/test/java/DataminerPoolManagerTest.java b/src/test/java/DataminerPoolManagerTest.java index 33e2aea..3396c66 100644 --- a/src/test/java/DataminerPoolManagerTest.java +++ b/src/test/java/DataminerPoolManagerTest.java @@ -220,7 +220,7 @@ public class DataminerPoolManagerTest { ensemble.addDependency(d); algorithms.addAlgorithm(ensemble); - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID()); + new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); } diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java index d2af50f..f1866d0 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/DataminerPoolManagerTest.java @@ -220,7 +220,7 @@ public class DataminerPoolManagerTest { ensemble.addDependency(d); algorithms.addAlgorithm(ensemble); - new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID()); + new DataminerPoolManager().addAlgorithmsToVRE(algorithms, "/gcube/devNext/NextNext", "test"+UUID.randomUUID(), false); }