From ec7d888eb13dbae2aac9385f54ba155f1b96fd27 Mon Sep 17 00:00:00 2001 From: Gabriele Giammatteo Date: Tue, 9 May 2017 16:46:52 +0000 Subject: [PATCH] refactoring (part 2) git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148423 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../poolmanager/ansible/AnsibleWorker.java | 157 ------------------ .../service/DataminerPoolManager.java | 41 ++--- .../poolmanager/util/SVNUpdater.java | 110 ++++++++++++ 3 files changed, 127 insertions(+), 181 deletions(-) create mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index efc0677..bb57266 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -140,163 +140,6 @@ public class AnsibleWorker { return -1; ***REMOVED*** - - - public void apply(AlgorithmSet as, PrintStream ps, boolean test) - throws IOException, InterruptedException, SVNException { - ***REMOVED*** TODO execute the playbook and return output - System.out.println(this.getWorkdir()); - try { - Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath() - + " " + this.getPlaybookFile().getAbsolutePath()); - - inheritIO(p.getInputStream(), ps); - inheritIO(p.getErrorStream(), ps); - - - int exitValue = p.waitFor(); - if (exitValue == 0) { - if (test) { - for (Algorithm algo : as.getAlgorithms()) { - - for (Dependency d : algo.getDependencies()) { - - if (d.getType().equals("os")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.updateSVN("test_r_deb_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("cran")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.updateSVN("test_r_cran_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("github")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.updateSVN("test_r_github_pkgs.txt", ls); - ***REMOVED*** - ***REMOVED*** - - ***REMOVED*** - - ***REMOVED*** - else if(!test){ - for (Algorithm algo : as.getAlgorithms()) { - - for (Dependency d : algo.getDependencies()) { - - if (d.getType().equals("os")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.updateSVN("r_deb_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("cran")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.updateSVN("r_cran_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("github")) { - List ls = new LinkedList(); - ls.add(d.getName()); - this.updateSVN("r_github_pkgs.txt", ls); - ***REMOVED*** - ***REMOVED*** - - ***REMOVED*** - ***REMOVED*** - - for (Algorithm algo : as.getAlgorithms()) { - DataminerPoolManager a = new DataminerPoolManager(); - a.addAlgToIs(algo); - ***REMOVED*** - ***REMOVED*** - - - ***REMOVED*** catch (IOException e) { - e.printStackTrace(); - ***REMOVED*** -***REMOVED*** - - - - private SVNRepository getSvnRepository(String url) throws SVNException { - SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); - ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); - repository.setAuthenticationManager(authManager); - ***REMOVED***System.out.println(repository.getLocation()); - - return repository; -***REMOVED*** - - - - public List updateSVN(String file, List ldep) throws SVNException, IOException { - final SVNRepository svnRepository = this.getSvnRepository( - "https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); - try { - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - List aa = this.checkMatch(lines, ldep); - Collections.sort(aa); - - final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); - - byte[] originalContents = byteArrayOutputStream.toByteArray(); - - final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); - commitEditor.openRoot(-1); - commitEditor.openFile(file, -1); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - for (String line : aa) { - baos.write(line.getBytes()); - baos.write("\n".getBytes()); - ***REMOVED*** - byte[] bytes = baos.toByteArray(); - - commitEditor.applyTextDelta(file,md5(originalContents)); - - final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, - new ByteArrayInputStream(bytes), commitEditor, true); - commitEditor.closeFile(file, checksum); - commitEditor.closeEdit(); - return aa; - - ***REMOVED*** finally { - svnRepository.closeSession(); - ***REMOVED*** -***REMOVED*** - - - - public static String md5(byte[] contents) { - final byte[] tmp = new byte[1024]; - final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5"); - try { - while (checksumStream.read(tmp) > 0) { - ***REMOVED*** - ***REMOVED*** - return checksumStream.getDigest(); - ***REMOVED*** catch (IOException e) { - ***REMOVED***never happens - e.printStackTrace(); - return null; - ***REMOVED*** finally { - SVNFileUtil.closeFile(checksumStream); - ***REMOVED*** -***REMOVED*** - - public List checkMatch(String[] lines, List ls) { - Set ss = new HashSet(ls); - ss.addAll(Arrays.asList(lines)); - return new ArrayList<>(ss); -***REMOVED*** - - - private static void inheritIO(final InputStream src, final PrintStream dest) { new Thread(new Runnable() { public void run() { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index cc895b0..fc162ef 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -8,6 +8,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*; +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.informationsystem.publisher.AdvancedScopedPublisher; import org.gcube.informationsystem.publisher.RegistryPublisherFactory; import org.gcube.informationsystem.publisher.ScopedPublisher; @@ -33,6 +34,19 @@ public class DataminerPoolManager { private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class); + private SVNUpdater svnUpdater; + + public DataminerPoolManager(){ + try { + this.svnUpdater = new SVNUpdater( + "https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); + ***REMOVED*** catch (SVNException e) { + e.printStackTrace(); + ***REMOVED*** +***REMOVED*** + + + public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException { Cluster cluster = new Cluster(); @@ -66,17 +80,17 @@ public class DataminerPoolManager { if (d.getType().equals("os")) { List ls = new LinkedList(); ls.add(d.getName()); - this.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls); + this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls); ***REMOVED*** if (d.getType().equals("cran")) { List ls = new LinkedList(); ls.add(d.getName()); - this.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls); + this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls); ***REMOVED*** if (d.getType().equals("github")) { List ls = new LinkedList(); ls.add(d.getName()); - this.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls); + this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls); ***REMOVED*** ***REMOVED*** ***REMOVED*** @@ -154,13 +168,6 @@ public class DataminerPoolManager { return bufferScript; ***REMOVED*** - /** - * Publish the given algorithm in the given VRE - * - * @param algorithmName - * @param vre - * - */ public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException { ***REMOVED*** create a fake algorithm set final AlgorithmSet algoSet = new AlgorithmSet(); @@ -391,18 +398,4 @@ public class DataminerPoolManager { ***REMOVED*** - public List updateSVN(String file, List ldep) throws SVNException, IOException { - ***REMOVED*** TODO Auto-generated method stub - return null; -***REMOVED*** - - - - - - - - - - ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java new file mode 100644 index 0000000..21c8a90 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -0,0 +1,110 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +import org.tmatesoft.svn.core.SVNException; +import org.tmatesoft.svn.core.SVNURL; +import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; +import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; +import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream; +import org.tmatesoft.svn.core.io.ISVNEditor; +import org.tmatesoft.svn.core.io.SVNRepository; +import org.tmatesoft.svn.core.io.SVNRepositoryFactory; +import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator; +import org.tmatesoft.svn.core.wc.SVNWCUtil; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +***REMOVED*** +import java.util.*; + +/** + * Created by ggiammat on 5/9/17. + */ +public class SVNUpdater { + + + private SVNRepository svnRepository; + + public SVNUpdater(String rootURL) throws SVNException { + final SVNRepository svnRepository = this.getSvnRepository( + rootURL); + + ***REMOVED*** + + + private SVNRepository getSvnRepository(String url) throws SVNException { + SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); + ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); + repository.setAuthenticationManager(authManager); + ***REMOVED***System.out.println(repository.getLocation()); + + return repository; + ***REMOVED*** + + + + + public List updateSVN(String file, List ldep) throws SVNException, IOException { + try { + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + List aa = this.checkMatch(lines, ldep); + Collections.sort(aa); + + final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + + byte[] originalContents = byteArrayOutputStream.toByteArray(); + + final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); + commitEditor.openRoot(-1); + commitEditor.openFile(file, -1); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + for (String line : aa) { + baos.write(line.getBytes()); + baos.write("\n".getBytes()); + ***REMOVED*** + byte[] bytes = baos.toByteArray(); + + commitEditor.applyTextDelta(file,md5(originalContents)); + + final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, + new ByteArrayInputStream(bytes), commitEditor, true); + commitEditor.closeFile(file, checksum); + commitEditor.closeEdit(); + return aa; + + ***REMOVED*** finally { + svnRepository.closeSession(); + ***REMOVED*** + ***REMOVED*** + + + + + public static String md5(byte[] contents) { + final byte[] tmp = new byte[1024]; + final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5"); + try { + while (checksumStream.read(tmp) > 0) { + ***REMOVED*** + ***REMOVED*** + return checksumStream.getDigest(); + ***REMOVED*** catch (IOException e) { + ***REMOVED***never happens + e.printStackTrace(); + return null; + ***REMOVED*** finally { + SVNFileUtil.closeFile(checksumStream); + ***REMOVED*** + ***REMOVED*** + + public List checkMatch(String[] lines, List ls) { + Set ss = new HashSet(ls); + ss.addAll(Arrays.asList(lines)); + return new ArrayList<>(ss); + ***REMOVED*** + + +***REMOVED***