refactoring (part 2)

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148423 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gabriele Giammatteo 2017-05-09 16:46:52 +00:00
parent a7ddc72bc8
commit 057dc2c4d6
3 changed files with 127 additions and 181 deletions

View File

@ -140,163 +140,6 @@ public class AnsibleWorker ***REMOVED***
return -1;
***REMOVED***
public void apply(AlgorithmSet as, PrintStream ps, boolean test)
throws IOException, InterruptedException, SVNException ***REMOVED***
***REMOVED*** TODO execute the playbook and return output
System.out.println(this.getWorkdir());
try ***REMOVED***
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath()
+ " " + this.getPlaybookFile().getAbsolutePath());
inheritIO(p.getInputStream(), ps);
inheritIO(p.getErrorStream(), ps);
int exitValue = p.waitFor();
if (exitValue == 0) ***REMOVED***
if (test) ***REMOVED***
for (Algorithm algo : as.getAlgorithms()) ***REMOVED***
for (Dependency d : algo.getDependencies()) ***REMOVED***
if (d.getType().equals("os")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("test_r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("test_r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("github")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("test_r_github_pkgs.txt", ls);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
else if(!test)***REMOVED***
for (Algorithm algo : as.getAlgorithms()) ***REMOVED***
for (Dependency d : algo.getDependencies()) ***REMOVED***
if (d.getType().equals("os")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("github")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN("r_github_pkgs.txt", ls);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
for (Algorithm algo : as.getAlgorithms()) ***REMOVED***
DataminerPoolManager a = new DataminerPoolManager();
a.addAlgToIs(algo);
***REMOVED***
***REMOVED***
***REMOVED*** catch (IOException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***
private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED***
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
repository.setAuthenticationManager(authManager);
***REMOVED***System.out.println(repository.getLocation());
return repository;
***REMOVED***
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException ***REMOVED***
final SVNRepository svnRepository = this.getSvnRepository(
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
try ***REMOVED***
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> aa = this.checkMatch(lines, ldep);
Collections.sort(aa);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
byte[] originalContents = byteArrayOutputStream.toByteArray();
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : aa) ***REMOVED***
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file,md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
commitEditor.closeEdit();
return aa;
***REMOVED*** finally ***REMOVED***
svnRepository.closeSession();
***REMOVED***
***REMOVED***
public static String md5(byte[] contents) ***REMOVED***
final byte[] tmp = new byte[1024];
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
try ***REMOVED***
while (checksumStream.read(tmp) > 0) ***REMOVED***
***REMOVED***
***REMOVED***
return checksumStream.getDigest();
***REMOVED*** catch (IOException e) ***REMOVED***
***REMOVED***never happens
e.printStackTrace();
return null;
***REMOVED*** finally ***REMOVED***
SVNFileUtil.closeFile(checksumStream);
***REMOVED***
***REMOVED***
public List<String> checkMatch(String[] lines, List<String> ls) ***REMOVED***
Set<String> ss = new HashSet<String>(ls);
ss.addAll(Arrays.asList(lines));
return new ArrayList<>(ss);
***REMOVED***
private static void inheritIO(final InputStream src, final PrintStream dest) ***REMOVED***
new Thread(new Runnable() ***REMOVED***
public void run() ***REMOVED***

View File

@ -8,6 +8,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
import org.gcube.informationsystem.publisher.ScopedPublisher;
@ -33,6 +34,19 @@ public class DataminerPoolManager ***REMOVED***
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
private SVNUpdater svnUpdater;
public DataminerPoolManager()***REMOVED***
try ***REMOVED***
this.svnUpdater = new SVNUpdater(
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
***REMOVED*** catch (SVNException e) ***REMOVED***
e.printStackTrace();
***REMOVED***
***REMOVED***
***REMOVED***Algorithm algo) throws IOException, InterruptedException ***REMOVED***
Cluster cluster = new Cluster();
@ -66,17 +80,17 @@ public class DataminerPoolManager ***REMOVED***
if (d.getType().equals("os")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("github")) ***REMOVED***
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
***REMOVED***
***REMOVED***
***REMOVED***
@ -154,13 +168,6 @@ public class DataminerPoolManager ***REMOVED***
return bufferScript;
***REMOVED***
***REMOVED****
* Publish the given algorithm in the given VRE
*
* @param algorithmName
* @param vre
*
***REMOVED***
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException ***REMOVED***
***REMOVED*** create a fake algorithm set
final AlgorithmSet algoSet = new AlgorithmSet();
@ -391,18 +398,4 @@ public class DataminerPoolManager ***REMOVED***
***REMOVED***
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***

View File

@ -0,0 +1,110 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
***REMOVED***
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
import org.tmatesoft.svn.core.io.ISVNEditor;
import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
import org.tmatesoft.svn.core.wc.SVNWCUtil;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
***REMOVED***
import java.util.*;
/**
* Created by ggiammat on 5/9/17.
*/
public class SVNUpdater ***REMOVED***
private SVNRepository svnRepository;
public SVNUpdater(String rootURL) throws SVNException ***REMOVED***
final SVNRepository svnRepository = this.getSvnRepository(
rootURL);
***REMOVED***
private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED***
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
repository.setAuthenticationManager(authManager);
***REMOVED***System.out.println(repository.getLocation());
return repository;
***REMOVED***
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException ***REMOVED***
try ***REMOVED***
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> aa = this.checkMatch(lines, ldep);
Collections.sort(aa);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
byte[] originalContents = byteArrayOutputStream.toByteArray();
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : aa) ***REMOVED***
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file,md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
commitEditor.closeEdit();
return aa;
***REMOVED*** finally ***REMOVED***
svnRepository.closeSession();
***REMOVED***
***REMOVED***
public static String md5(byte[] contents) ***REMOVED***
final byte[] tmp = new byte[1024];
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
try ***REMOVED***
while (checksumStream.read(tmp) > 0) ***REMOVED***
***REMOVED***
***REMOVED***
return checksumStream.getDigest();
***REMOVED*** catch (IOException e) ***REMOVED***
***REMOVED***never happens
e.printStackTrace();
return null;
***REMOVED*** finally ***REMOVED***
SVNFileUtil.closeFile(checksumStream);
***REMOVED***
***REMOVED***
public List<String> checkMatch(String[] lines, List<String> ls) ***REMOVED***
Set<String> ss = new HashSet<String>(ls);
ss.addAll(Arrays.asList(lines));
return new ArrayList<>(ss);
***REMOVED***
***REMOVED***