refactoring (part 2)
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148423 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
dc2061ead6
commit
ec7d888eb1
|
@ -140,163 +140,6 @@ public class AnsibleWorker {
|
||||||
return -1;
|
return -1;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public void apply(AlgorithmSet as, PrintStream ps, boolean test)
|
|
||||||
throws IOException, InterruptedException, SVNException {
|
|
||||||
***REMOVED*** TODO execute the playbook and return output
|
|
||||||
System.out.println(this.getWorkdir());
|
|
||||||
try {
|
|
||||||
Process p = Runtime.getRuntime().exec("ansible-playbook -v -i " + this.getInventoryFile().getAbsolutePath()
|
|
||||||
+ " " + this.getPlaybookFile().getAbsolutePath());
|
|
||||||
|
|
||||||
inheritIO(p.getInputStream(), ps);
|
|
||||||
inheritIO(p.getErrorStream(), ps);
|
|
||||||
|
|
||||||
|
|
||||||
int exitValue = p.waitFor();
|
|
||||||
if (exitValue == 0) {
|
|
||||||
if (test) {
|
|
||||||
for (Algorithm algo : as.getAlgorithms()) {
|
|
||||||
|
|
||||||
for (Dependency d : algo.getDependencies()) {
|
|
||||||
|
|
||||||
if (d.getType().equals("os")) {
|
|
||||||
List<String> ls = new LinkedList<String>();
|
|
||||||
ls.add(d.getName());
|
|
||||||
this.updateSVN("test_r_deb_pkgs.txt", ls);
|
|
||||||
***REMOVED***
|
|
||||||
if (d.getType().equals("cran")) {
|
|
||||||
List<String> ls = new LinkedList<String>();
|
|
||||||
ls.add(d.getName());
|
|
||||||
this.updateSVN("test_r_cran_pkgs.txt", ls);
|
|
||||||
***REMOVED***
|
|
||||||
if (d.getType().equals("github")) {
|
|
||||||
List<String> ls = new LinkedList<String>();
|
|
||||||
ls.add(d.getName());
|
|
||||||
this.updateSVN("test_r_github_pkgs.txt", ls);
|
|
||||||
***REMOVED***
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
***REMOVED***
|
|
||||||
else if(!test){
|
|
||||||
for (Algorithm algo : as.getAlgorithms()) {
|
|
||||||
|
|
||||||
for (Dependency d : algo.getDependencies()) {
|
|
||||||
|
|
||||||
if (d.getType().equals("os")) {
|
|
||||||
List<String> ls = new LinkedList<String>();
|
|
||||||
ls.add(d.getName());
|
|
||||||
this.updateSVN("r_deb_pkgs.txt", ls);
|
|
||||||
***REMOVED***
|
|
||||||
if (d.getType().equals("cran")) {
|
|
||||||
List<String> ls = new LinkedList<String>();
|
|
||||||
ls.add(d.getName());
|
|
||||||
this.updateSVN("r_cran_pkgs.txt", ls);
|
|
||||||
***REMOVED***
|
|
||||||
if (d.getType().equals("github")) {
|
|
||||||
List<String> ls = new LinkedList<String>();
|
|
||||||
ls.add(d.getName());
|
|
||||||
this.updateSVN("r_github_pkgs.txt", ls);
|
|
||||||
***REMOVED***
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
***REMOVED***
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
for (Algorithm algo : as.getAlgorithms()) {
|
|
||||||
DataminerPoolManager a = new DataminerPoolManager();
|
|
||||||
a.addAlgToIs(algo);
|
|
||||||
***REMOVED***
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
|
|
||||||
***REMOVED*** catch (IOException e) {
|
|
||||||
e.printStackTrace();
|
|
||||||
***REMOVED***
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private SVNRepository getSvnRepository(String url) throws SVNException {
|
|
||||||
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
|
||||||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
|
||||||
repository.setAuthenticationManager(authManager);
|
|
||||||
***REMOVED***System.out.println(repository.getLocation());
|
|
||||||
|
|
||||||
return repository;
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
|
|
||||||
final SVNRepository svnRepository = this.getSvnRepository(
|
|
||||||
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
|
||||||
try {
|
|
||||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
|
||||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
|
||||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
|
||||||
List<String> aa = this.checkMatch(lines, ldep);
|
|
||||||
Collections.sort(aa);
|
|
||||||
|
|
||||||
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
|
||||||
|
|
||||||
byte[] originalContents = byteArrayOutputStream.toByteArray();
|
|
||||||
|
|
||||||
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
|
|
||||||
commitEditor.openRoot(-1);
|
|
||||||
commitEditor.openFile(file, -1);
|
|
||||||
|
|
||||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
||||||
|
|
||||||
for (String line : aa) {
|
|
||||||
baos.write(line.getBytes());
|
|
||||||
baos.write("\n".getBytes());
|
|
||||||
***REMOVED***
|
|
||||||
byte[] bytes = baos.toByteArray();
|
|
||||||
|
|
||||||
commitEditor.applyTextDelta(file,md5(originalContents));
|
|
||||||
|
|
||||||
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
|
||||||
new ByteArrayInputStream(bytes), commitEditor, true);
|
|
||||||
commitEditor.closeFile(file, checksum);
|
|
||||||
commitEditor.closeEdit();
|
|
||||||
return aa;
|
|
||||||
|
|
||||||
***REMOVED*** finally {
|
|
||||||
svnRepository.closeSession();
|
|
||||||
***REMOVED***
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public static String md5(byte[] contents) {
|
|
||||||
final byte[] tmp = new byte[1024];
|
|
||||||
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
|
|
||||||
try {
|
|
||||||
while (checksumStream.read(tmp) > 0) {
|
|
||||||
***REMOVED***
|
|
||||||
***REMOVED***
|
|
||||||
return checksumStream.getDigest();
|
|
||||||
***REMOVED*** catch (IOException e) {
|
|
||||||
***REMOVED***never happens
|
|
||||||
e.printStackTrace();
|
|
||||||
return null;
|
|
||||||
***REMOVED*** finally {
|
|
||||||
SVNFileUtil.closeFile(checksumStream);
|
|
||||||
***REMOVED***
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
public List<String> checkMatch(String[] lines, List<String> ls) {
|
|
||||||
Set<String> ss = new HashSet<String>(ls);
|
|
||||||
ss.addAll(Arrays.asList(lines));
|
|
||||||
return new ArrayList<>(ss);
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private static void inheritIO(final InputStream src, final PrintStream dest) {
|
private static void inheritIO(final InputStream src, final PrintStream dest) {
|
||||||
new Thread(new Runnable() {
|
new Thread(new Runnable() {
|
||||||
public void run() {
|
public void run() {
|
||||||
|
|
|
@ -8,6 +8,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansible.AnsibleWorker;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
|
import org.gcube.dataanalysis.dataminer.poolmanager.clients.HAProxy;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.*;
|
||||||
|
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||||
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
|
import org.gcube.informationsystem.publisher.AdvancedScopedPublisher;
|
||||||
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
import org.gcube.informationsystem.publisher.RegistryPublisherFactory;
|
||||||
import org.gcube.informationsystem.publisher.ScopedPublisher;
|
import org.gcube.informationsystem.publisher.ScopedPublisher;
|
||||||
|
@ -33,6 +34,19 @@ public class DataminerPoolManager {
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(DataminerPoolManager.class);
|
||||||
|
|
||||||
|
|
||||||
|
private SVNUpdater svnUpdater;
|
||||||
|
|
||||||
|
public DataminerPoolManager(){
|
||||||
|
try {
|
||||||
|
this.svnUpdater = new SVNUpdater(
|
||||||
|
"https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
||||||
|
***REMOVED*** catch (SVNException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
***REMOVED***
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
|
public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
|
||||||
|
|
||||||
Cluster cluster = new Cluster();
|
Cluster cluster = new Cluster();
|
||||||
|
@ -66,17 +80,17 @@ public class DataminerPoolManager {
|
||||||
if (d.getType().equals("os")) {
|
if (d.getType().equals("os")) {
|
||||||
List<String> ls = new LinkedList<String>();
|
List<String> ls = new LinkedList<String>();
|
||||||
ls.add(d.getName());
|
ls.add(d.getName());
|
||||||
this.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_deb_pkgs.txt", ls);
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
if (d.getType().equals("cran")) {
|
if (d.getType().equals("cran")) {
|
||||||
List<String> ls = new LinkedList<String>();
|
List<String> ls = new LinkedList<String>();
|
||||||
ls.add(d.getName());
|
ls.add(d.getName());
|
||||||
this.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_cran_pkgs.txt", ls);
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
if (d.getType().equals("github")) {
|
if (d.getType().equals("github")) {
|
||||||
List<String> ls = new LinkedList<String>();
|
List<String> ls = new LinkedList<String>();
|
||||||
ls.add(d.getName());
|
ls.add(d.getName());
|
||||||
this.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
|
this.svnUpdater.updateSVN(stagingVRE ? "test_": "" + "r_github_pkgs.txt", ls);
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
@ -154,13 +168,6 @@ public class DataminerPoolManager {
|
||||||
return bufferScript;
|
return bufferScript;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
/**
|
|
||||||
* Publish the given algorithm in the given VRE
|
|
||||||
*
|
|
||||||
* @param algorithmName
|
|
||||||
* @param vre
|
|
||||||
*
|
|
||||||
*/
|
|
||||||
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
public String addAlgorithmToVRE(Algorithm algorithm, final String vre, /*final boolean updateSVN*/ final boolean test) throws IOException {
|
||||||
***REMOVED*** create a fake algorithm set
|
***REMOVED*** create a fake algorithm set
|
||||||
final AlgorithmSet algoSet = new AlgorithmSet();
|
final AlgorithmSet algoSet = new AlgorithmSet();
|
||||||
|
@ -391,18 +398,4 @@ public class DataminerPoolManager {
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
|
|
||||||
***REMOVED*** TODO Auto-generated method stub
|
|
||||||
return null;
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
|
@ -0,0 +1,110 @@
|
||||||
|
package org.gcube.dataanalysis.dataminer.poolmanager.util;
|
||||||
|
|
||||||
|
import org.tmatesoft.svn.core.SVNException;
|
||||||
|
import org.tmatesoft.svn.core.SVNURL;
|
||||||
|
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
|
||||||
|
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
|
||||||
|
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
|
||||||
|
import org.tmatesoft.svn.core.io.ISVNEditor;
|
||||||
|
import org.tmatesoft.svn.core.io.SVNRepository;
|
||||||
|
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
|
||||||
|
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
|
||||||
|
import org.tmatesoft.svn.core.wc.SVNWCUtil;
|
||||||
|
|
||||||
|
import java.io.ByteArrayInputStream;
|
||||||
|
import java.io.ByteArrayOutputStream;
|
||||||
|
***REMOVED***
|
||||||
|
import java.util.*;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Created by ggiammat on 5/9/17.
|
||||||
|
*/
|
||||||
|
public class SVNUpdater {
|
||||||
|
|
||||||
|
|
||||||
|
private SVNRepository svnRepository;
|
||||||
|
|
||||||
|
public SVNUpdater(String rootURL) throws SVNException {
|
||||||
|
final SVNRepository svnRepository = this.getSvnRepository(
|
||||||
|
rootURL);
|
||||||
|
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
private SVNRepository getSvnRepository(String url) throws SVNException {
|
||||||
|
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
||||||
|
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
||||||
|
repository.setAuthenticationManager(authManager);
|
||||||
|
***REMOVED***System.out.println(repository.getLocation());
|
||||||
|
|
||||||
|
return repository;
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public List<String> updateSVN(String file, List<String> ldep) throws SVNException, IOException {
|
||||||
|
try {
|
||||||
|
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||||
|
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||||
|
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||||
|
List<String> aa = this.checkMatch(lines, ldep);
|
||||||
|
Collections.sort(aa);
|
||||||
|
|
||||||
|
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
||||||
|
|
||||||
|
byte[] originalContents = byteArrayOutputStream.toByteArray();
|
||||||
|
|
||||||
|
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
|
||||||
|
commitEditor.openRoot(-1);
|
||||||
|
commitEditor.openFile(file, -1);
|
||||||
|
|
||||||
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||||
|
|
||||||
|
for (String line : aa) {
|
||||||
|
baos.write(line.getBytes());
|
||||||
|
baos.write("\n".getBytes());
|
||||||
|
***REMOVED***
|
||||||
|
byte[] bytes = baos.toByteArray();
|
||||||
|
|
||||||
|
commitEditor.applyTextDelta(file,md5(originalContents));
|
||||||
|
|
||||||
|
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
||||||
|
new ByteArrayInputStream(bytes), commitEditor, true);
|
||||||
|
commitEditor.closeFile(file, checksum);
|
||||||
|
commitEditor.closeEdit();
|
||||||
|
return aa;
|
||||||
|
|
||||||
|
***REMOVED*** finally {
|
||||||
|
svnRepository.closeSession();
|
||||||
|
***REMOVED***
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public static String md5(byte[] contents) {
|
||||||
|
final byte[] tmp = new byte[1024];
|
||||||
|
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
|
||||||
|
try {
|
||||||
|
while (checksumStream.read(tmp) > 0) {
|
||||||
|
***REMOVED***
|
||||||
|
***REMOVED***
|
||||||
|
return checksumStream.getDigest();
|
||||||
|
***REMOVED*** catch (IOException e) {
|
||||||
|
***REMOVED***never happens
|
||||||
|
e.printStackTrace();
|
||||||
|
return null;
|
||||||
|
***REMOVED*** finally {
|
||||||
|
SVNFileUtil.closeFile(checksumStream);
|
||||||
|
***REMOVED***
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public List<String> checkMatch(String[] lines, List<String> ls) {
|
||||||
|
Set<String> ss = new HashSet<String>(ls);
|
||||||
|
ss.addAll(Arrays.asList(lines));
|
||||||
|
return new ArrayList<>(ss);
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
***REMOVED***
|
Loading…
Reference in New Issue