427 lines
16 KiB
Java
427 lines
16 KiB
Java
package org.gcube.dataanalysis.dataminer.poolmanager.util;
|
|
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
|
|
import org.tmatesoft.svn.core.SVNException;
|
|
import org.tmatesoft.svn.core.SVNURL;
|
|
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
|
|
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
|
|
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
|
|
import org.tmatesoft.svn.core.io.ISVNEditor;
|
|
import org.tmatesoft.svn.core.io.SVNRepository;
|
|
import org.tmatesoft.svn.core.io.SVNRepositoryFactory;
|
|
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
|
|
import org.tmatesoft.svn.core.wc.SVNWCUtil;
|
|
|
|
import java.io.ByteArrayInputStream;
|
|
import java.io.ByteArrayOutputStream;
|
|
***REMOVED***
|
|
import java.util.*;
|
|
|
|
/**
|
|
* Created by ggiammat on 5/9/17.
|
|
*/
|
|
public class SVNUpdater {
|
|
|
|
private SVNRepository svnRepository;
|
|
private ServiceConfiguration configuration;
|
|
|
|
public SVNUpdater(ServiceConfiguration configuration) throws SVNException {
|
|
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
|
|
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
|
this.svnRepository.setAuthenticationManager(authManager);
|
|
this.configuration = configuration;
|
|
***REMOVED***
|
|
|
|
***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) {
|
|
***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies());
|
|
***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies());
|
|
***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
|
|
***REMOVED******REMOVED***
|
|
|
|
public String getRProtoDependencyFile(String language) {
|
|
switch (language) {
|
|
case "R":
|
|
return this.configuration.getSVNRProtoCRANDepsList();
|
|
case "R-blackbox":
|
|
return this.configuration.getSVNRProtoRBDepsList();
|
|
case "Java":
|
|
return this.configuration.getSVNRProtoJavaDepsList();
|
|
case "Knime-Workflow":
|
|
return this.configuration.getSVNRProtoKWDepsList();
|
|
case "Linux-compiled":
|
|
return this.configuration.getSVNRProtoLinuxCompiledDepsList();
|
|
case "Octave":
|
|
return this.configuration.getSVNRProtoOctaveDepsList();
|
|
case "Python":
|
|
return this.configuration.getSVNRProtoPythonDepsList();
|
|
case "Pre-Installed":
|
|
return this.configuration.getSVNRProtoPreInstalledDepsList();
|
|
case "Windows-compiled":
|
|
return this.configuration.getSVNRProtoWCDepsList();
|
|
default:
|
|
return null;
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
|
|
|
|
public String getProdDependencyFile(String language) {
|
|
switch (language) {
|
|
case "R":
|
|
return this.configuration.getSVNRProdCRANDepsList();
|
|
case "R-blackbox":
|
|
return this.configuration.getSVNRProdRBDepsList();
|
|
case "Java":
|
|
return this.configuration.getSVNRProdJavaDepsList();
|
|
case "Knime-Workflow":
|
|
return this.configuration.getSVNRProdKWDepsList();
|
|
case "Linux-compiled":
|
|
return this.configuration.getSVNRProdLinuxCompiledDepsList();
|
|
case "Octave":
|
|
return this.configuration.getSVNRProdOctaveDepsList();
|
|
case "Python":
|
|
return this.configuration.getSVNRProdPythonDepsList();
|
|
case "Pre-Installed":
|
|
return this.configuration.getSVNRProdPreInstalledDepsList();
|
|
case "Windows-compiled":
|
|
return this.configuration.getSVNRProdWCDepsList();
|
|
default:
|
|
return null;
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
|
|
|
|
public void readRPRotoDeps(Algorithm algorithm) throws SVNException {
|
|
if (algorithm.getLanguage().equals("R")) {
|
|
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("R-blackbox")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Java")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Knime-Workflow")) {
|
|
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Linux-compiled")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Octave")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Python")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Windows-compiled")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Pre-Installed")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
public void readProdDeps(Algorithm algorithm) throws SVNException {
|
|
if (algorithm.getLanguage().equals("R")) {
|
|
this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("R-blackbox")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Java")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Knime-Workflow")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Linux-compiled")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Octave")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Python")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Windows-compiled")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
if (algorithm.getLanguage().equals("Pre-Installed")) {
|
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
***REMOVED*** public void updateProdDeps(Algorithm algorithm) {
|
|
***REMOVED*** this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
|
|
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
|
|
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
|
|
***REMOVED******REMOVED***
|
|
|
|
public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) {
|
|
this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env);
|
|
***REMOVED***
|
|
|
|
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) {
|
|
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env);
|
|
***REMOVED***
|
|
|
|
public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env) {
|
|
try {
|
|
System.out.println("Updating algorithm list: " + file);
|
|
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
|
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
|
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
|
|
|
***REMOVED*** check if the algorithm is already in the list (match the class
|
|
***REMOVED*** name)
|
|
for (String l : lines) {
|
|
if (l.contains(algorithm.getClazz())) {
|
|
System.out.println("Not updating algorithm list beacuse already present");
|
|
return;
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
***REMOVED*** the algorithm is not in the list. Add it
|
|
List<String> newContent = new LinkedList<>(Arrays.asList(lines));
|
|
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env));
|
|
***REMOVED*** Collections.sort(newContent);
|
|
|
|
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
|
|
|
byte[] originalContents = byteArrayOutputStream.toByteArray();
|
|
|
|
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update algorithm list", null);
|
|
commitEditor.openRoot(-1);
|
|
commitEditor.openFile(file, -1);
|
|
|
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
|
|
for (String line : newContent) {
|
|
baos.write(line.getBytes());
|
|
baos.write("\n".getBytes());
|
|
***REMOVED***
|
|
byte[] bytes = baos.toByteArray();
|
|
|
|
commitEditor.applyTextDelta(file, md5(originalContents));
|
|
|
|
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
|
new ByteArrayInputStream(bytes), commitEditor, true);
|
|
commitEditor.closeFile(file, checksum);
|
|
commitEditor.closeEdit();
|
|
|
|
***REMOVED*** catch (Exception ex) {
|
|
ex.printStackTrace();
|
|
***REMOVED***
|
|
|
|
finally {
|
|
svnRepository.closeSession();
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env) {
|
|
StringBuffer sb = new StringBuffer("| ");
|
|
sb.append(algorithm.getName() + " | ");
|
|
sb.append(algorithm.getFullname() + " | ");
|
|
sb.append(algorithm.getCategory() + " | ");
|
|
sb.append(env + " | ");
|
|
sb.append("<notextile>./addAlgorithm.sh " + algorithm.getName() + " " + algorithm.getCategory() + " "
|
|
+ algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N "
|
|
+ algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
|
|
sb.append("none |");
|
|
return sb.toString();
|
|
***REMOVED***
|
|
|
|
|
|
public Collection<String> getUndefinedDependencies(String file, Collection<Dependency> deps) throws SVNException {
|
|
***REMOVED*** SendMail sm = new SendMail();
|
|
***REMOVED*** NotificationHelper nh = new NotificationHelper();
|
|
|
|
|
|
System.out.println("Checking dependencies list: " + file);
|
|
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
|
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
|
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
|
List<String> validDependencies = Arrays.asList(lines);
|
|
|
|
|
|
|
|
List<String> undefined = new LinkedList<String>();
|
|
|
|
|
|
|
|
for(Dependency d: deps){
|
|
String depName = d.getName();
|
|
if(!validDependencies.contains(depName)){
|
|
undefined.add(depName);
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
return undefined;
|
|
***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED*** for (String a : lines) {
|
|
***REMOVED*** for (String b : ldep) {
|
|
***REMOVED*** if (b.equals(a)) {
|
|
***REMOVED*** System.out.println("The following dependency is correctly written: " + b);
|
|
***REMOVED*** ***REMOVED*** else
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED*** boolean check = false;
|
|
***REMOVED*** try {
|
|
***REMOVED*** System.out.println("Checking dependencies list: " + file);
|
|
***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
|
***REMOVED*** svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
|
***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED*** if(deps.isEmpty()){
|
|
***REMOVED*** ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody());
|
|
***REMOVED*** ***REMOVED*** Exception e = new Exception("No dependency specified for this
|
|
***REMOVED*** ***REMOVED*** algorithm");
|
|
***REMOVED*** ***REMOVED*** throw e;
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** ***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) {
|
|
***REMOVED*** List<String> ldep = new LinkedList<>();
|
|
***REMOVED*** for (Dependency d : deps) {
|
|
***REMOVED*** ldep.add(d.getName());
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** for (String a : lines) {
|
|
***REMOVED*** for (String b : ldep) {
|
|
***REMOVED*** if (b.equals(a)) {
|
|
***REMOVED*** System.out.println("The following dependency is correctly written: " + b);
|
|
***REMOVED*** check = true;
|
|
***REMOVED*** ***REMOVED*** else
|
|
***REMOVED*** check = false;
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** ***REMOVED***
|
|
***REMOVED*** ***REMOVED*** catch (Exception a) {
|
|
***REMOVED*** a.getMessage();
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED*** return check;
|
|
|
|
***REMOVED***
|
|
|
|
public boolean checkIfAvaialable(String file, Collection<Dependency> deps) throws SVNException {
|
|
SendMail sm = new SendMail();
|
|
NotificationHelper nh = new NotificationHelper();
|
|
|
|
boolean check = false;
|
|
try {
|
|
System.out.println("Checking dependencies list: " + file);
|
|
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
|
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
|
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
|
|
|
***REMOVED*** if(deps.isEmpty()){
|
|
***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody());
|
|
***REMOVED*** Exception e = new Exception("No dependency specified for this
|
|
***REMOVED*** algorithm");
|
|
***REMOVED*** throw e;
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
|
|
***REMOVED*** else if (!deps.isEmpty()) {
|
|
List<String> ldep = new LinkedList<>();
|
|
for (Dependency d : deps) {
|
|
ldep.add(d.getName());
|
|
***REMOVED***
|
|
for (String a : lines) {
|
|
for (String b : ldep) {
|
|
if (b.equals(a)) {
|
|
System.out.println("The following dependency is correctly written: " + b);
|
|
check = true;
|
|
***REMOVED*** else
|
|
check = false;
|
|
|
|
***REMOVED***
|
|
***REMOVED***
|
|
***REMOVED*** ***REMOVED***
|
|
***REMOVED*** catch (Exception a) {
|
|
a.getMessage();
|
|
***REMOVED***
|
|
|
|
return check;
|
|
|
|
***REMOVED***
|
|
|
|
public void updateSVN(String file, Collection<Dependency> deps) {
|
|
try {
|
|
System.out.println("Updating dependencies list: " + file);
|
|
|
|
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
|
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
|
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
|
|
|
List<String> ldep = new LinkedList<>();
|
|
for (Dependency d : deps) {
|
|
ldep.add(d.getName());
|
|
***REMOVED***
|
|
List<String> aa = this.checkMatch(lines, ldep);
|
|
Collections.sort(aa);
|
|
|
|
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
|
|
|
byte[] originalContents = byteArrayOutputStream.toByteArray();
|
|
|
|
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
|
|
commitEditor.openRoot(-1);
|
|
commitEditor.openFile(file, -1);
|
|
|
|
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
|
|
|
for (String line : aa) {
|
|
baos.write(line.getBytes());
|
|
baos.write("\n".getBytes());
|
|
***REMOVED***
|
|
byte[] bytes = baos.toByteArray();
|
|
|
|
commitEditor.applyTextDelta(file, md5(originalContents));
|
|
|
|
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
|
new ByteArrayInputStream(bytes), commitEditor, true);
|
|
commitEditor.closeFile(file, checksum);
|
|
commitEditor.closeEdit();
|
|
|
|
***REMOVED*** catch (Exception ex) {
|
|
ex.printStackTrace();
|
|
***REMOVED***
|
|
|
|
finally {
|
|
svnRepository.closeSession();
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
public static String md5(byte[] contents) {
|
|
final byte[] tmp = new byte[1024];
|
|
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents),
|
|
"md5");
|
|
try {
|
|
while (checksumStream.read(tmp) > 0) {
|
|
***REMOVED***
|
|
***REMOVED***
|
|
return checksumStream.getDigest();
|
|
***REMOVED*** catch (IOException e) {
|
|
***REMOVED*** never happens
|
|
e.printStackTrace();
|
|
return null;
|
|
***REMOVED*** finally {
|
|
SVNFileUtil.closeFile(checksumStream);
|
|
***REMOVED***
|
|
***REMOVED***
|
|
|
|
public List<String> checkMatch(String[] lines, List<String> ls) {
|
|
Set<String> ss = new HashSet<String>(ls);
|
|
ss.addAll(Arrays.asList(lines));
|
|
return new ArrayList<>(ss);
|
|
***REMOVED***
|
|
|
|
***REMOVED***
|