refactoring (part 2)
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148423 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gabriele Giammatteo
parent
dc2061ead6
commit
ec7d888eb1
|
|
@ -140,163 +140,6 @@ public class AnsibleWorker {
|
|||
return -1;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public void apply(AlgorithmSet as, PrintStream ps, boolean test)
|
||||
throws IOException, InterruptedException, SVNException {
|
||||
***REMOVED*** TODO execute the playbook and return output
|
||||
System.out.println(this.getWorkdir());
|
||||
try {
|
||||
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath()
|
||||
+ " " + this.getPlaybookFile().getAbsolutePath());
|
||||
|
||||
inheritIO(p.getInputStream(), ps);
|
||||
inheritIO(p.getErrorStream(), ps);
|
||||
|
||||
|
||||
int exitValue = p.waitFor();
|
||||
if (exitValue == 0) {
|
||||
if (test) {
|
||||
for (Algorithm algo : as.getAlgorithms()) {
|
||||
|
||||
for (Dependency d : algo.getDependencies()) {
|
||||
|
||||
if (d.getType().equals("os")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("test_r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("cran")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("test_r_cran_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("github")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("test_r_github_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
else if(!test){
|
||||
for (Algorithm algo : as.getAlgorithms()) {
|
||||
|
||||
for (Dependency d : algo.getDependencies()) {
|
||||
|
||||
if (d.getType().equals("os")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("cran")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_cran_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("github")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN("r_github_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
for (Algorithm algo : as.getAlgorithms()) {
|
||||
DataminerPoolManager a = new DataminerPoolManager();
|
||||
a.addAlgToIs(algo);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
***REMOVED*** catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
private SVNRepository getSvnRepository(String url) throws SVNException {
|
||||
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
||||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
||||
repository.setAuthenticationManager(authManager);
|
||||
***REMOVED***System.out.println(repository.getLocation());
|
||||
|
||||
return repository;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
|
||||
final SVNRepository svnRepository = this.getSvnRepository(
|
||||
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
||||
try {
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
List<String> aa = this.checkMatch(lines, ldep);
|
||||
Collections.sort(aa);
|
||||
|
||||
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
||||
|
||||
byte[] originalContents = byteArrayOutputStream.toByteArray();
|
||||
|
||||
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
|
||||
commitEditor.openRoot(-1);
|
||||
commitEditor.openFile(file, -1);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
|
||||
for (String line : aa) {
|
||||
baos.write(line.getBytes());
|
||||
baos.write("\n".getBytes());
|
||||
***REMOVED***
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
commitEditor.applyTextDelta(file,md5(originalContents));
|
||||
|
||||
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
||||
new ByteArrayInputStream(bytes), commitEditor, true);
|
||||
commitEditor.closeFile(file, checksum);
|
||||
commitEditor.closeEdit();
|
||||
return aa;
|
||||
|
||||
***REMOVED*** finally {
|
||||
svnRepository.closeSession();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public static String md5(byte[] contents) {
|
||||
final byte[] tmp = new byte[1024];
|
||||
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
|
||||
try {
|
||||
while (checksumStream.read(tmp) > 0) {
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return checksumStream.getDigest();
|
||||
***REMOVED*** catch (IOException e) {
|
||||
***REMOVED***never happens
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
***REMOVED*** finally {
|
||||
SVNFileUtil.closeFile(checksumStream);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public List<String> checkMatch(String[] lines, List<String> ls) {
|
||||
Set<String> ss = new HashSet<String>(ls);
|
||||
ss.addAll(Arrays.asList(lines));
|
||||
return new ArrayList<>(ss);
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
private static void inheritIO(final InputStream src, final PrintStream dest) {
|
||||
new Thread(new Runnable() {
|
||||
public void run() {
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
|
|||
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
|
||||
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
||||
import org.gcube.informationsystem.publisher.ScopedPublisher;
|
||||
|
|
@ -33,6 +34,19 @@ public class DataminerPoolManager {
|
|||
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
||||
|
||||
|
||||
private SVNUpdater svnUpdater;
|
||||
|
||||
public DataminerPoolManager(){
|
||||
try {
|
||||
this.svnUpdater = new SVNUpdater(
|
||||
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
||||
***REMOVED*** catch (SVNException e) {
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
|
||||
|
||||
Cluster cluster = new Cluster();
|
||||
|
|
@ -66,17 +80,17 @@ public class DataminerPoolManager {
|
|||
if (d.getType().equals("os")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
|
||||
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("cran")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
|
||||
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("github")) {
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
|
||||
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
|
@ -154,13 +168,6 @@ public class DataminerPoolManager {
|
|||
return bufferScript;
|
||||
***REMOVED***
|
||||
|
||||
/**
|
||||
* Publish the given algorithm in the given VRE
|
||||
*
|
||||
* @param algorithmName
|
||||
* @param vre
|
||||
*
|
||||
*/
|
||||
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
||||
***REMOVED*** create a fake algorithm set
|
||||
final AlgorithmSet algoSet = new AlgorithmSet();
|
||||
|
|
@ -391,18 +398,4 @@ public class DataminerPoolManager {
|
|||
***REMOVED***
|
||||
|
||||
|
||||
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
|
||||
***REMOVED*** TODO Auto-generated method stub
|
||||
return null;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.util;
|
||||
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
import org.tmatesoft.svn.core.SVNURL;
|
||||
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
|
||||
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
|
||||
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
|
||||
import org.tmatesoft.svn.core.io.ISVNEditor;
|
||||
import org.tmatesoft.svn.core.io.SVNRepository;
|
||||
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
|
||||
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
|
||||
import org.tmatesoft.svn.core.wc.SVNWCUtil;
|
||||
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.ByteArrayOutputStream;
|
||||
***REMOVED***
|
||||
import java.util.*;
|
||||
|
||||
/**
|
||||
* Created by ggiammat on 5/9/17.
|
||||
*/
|
||||
public class SVNUpdater {
|
||||
|
||||
|
||||
private SVNRepository svnRepository;
|
||||
|
||||
public SVNUpdater(String rootURL) throws SVNException {
|
||||
final SVNRepository svnRepository = this.getSvnRepository(
|
||||
rootURL);
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
private SVNRepository getSvnRepository(String url) throws SVNException {
|
||||
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
||||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
||||
repository.setAuthenticationManager(authManager);
|
||||
***REMOVED***System.out.println(repository.getLocation());
|
||||
|
||||
return repository;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
|
||||
try {
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
List<String> aa = this.checkMatch(lines, ldep);
|
||||
Collections.sort(aa);
|
||||
|
||||
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
||||
|
||||
byte[] originalContents = byteArrayOutputStream.toByteArray();
|
||||
|
||||
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
|
||||
commitEditor.openRoot(-1);
|
||||
commitEditor.openFile(file, -1);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
|
||||
for (String line : aa) {
|
||||
baos.write(line.getBytes());
|
||||
baos.write("\n".getBytes());
|
||||
***REMOVED***
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
commitEditor.applyTextDelta(file,md5(originalContents));
|
||||
|
||||
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
||||
new ByteArrayInputStream(bytes), commitEditor, true);
|
||||
commitEditor.closeFile(file, checksum);
|
||||
commitEditor.closeEdit();
|
||||
return aa;
|
||||
|
||||
***REMOVED*** finally {
|
||||
svnRepository.closeSession();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
public static String md5(byte[] contents) {
|
||||
final byte[] tmp = new byte[1024];
|
||||
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
|
||||
try {
|
||||
while (checksumStream.read(tmp) > 0) {
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return checksumStream.getDigest();
|
||||
***REMOVED*** catch (IOException e) {
|
||||
***REMOVED***never happens
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
***REMOVED*** finally {
|
||||
SVNFileUtil.closeFile(checksumStream);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public List<String> checkMatch(String[] lines, List<String> ls) {
|
||||
Set<String> ss = new HashSet<String>(ls);
|
||||
ss.addAll(Arrays.asList(lines));
|
||||
return new ArrayList<>(ss);
|
||||
***REMOVED***
|
||||
|
||||
|
||||
***REMOVED***
|
||||
Loading…
Reference in New Issue