dataminer-pool-manager/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java

580 lines
20 KiB
Java
Executable File

package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
***REMOVED***
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedList;
***REMOVED***
***REMOVED***
import java.util.TimeZone;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException;
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNCommitInfo;
import org.tmatesoft.svn.core.SVNErrorMessage;
***REMOVED***
import org.tmatesoft.svn.core.SVNNodeKind;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream;
import org.tmatesoft.svn.core.io.ISVNEditor;
import org.tmatesoft.svn.core.io.SVNRepository;
import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator;
/**
* Created by ggiammat on 5/9/17.
*/
public abstract class SVNUpdater ***REMOVED***
private SVNRepository svnRepository;
private Configuration configuration;
private Logger logger;
public SVNUpdater(Configuration configuration) throws SVNException ***REMOVED***
this.configuration = configuration;
this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository();
this.logger = LoggerFactory.getLogger(SVNUpdater.class);
***REMOVED***
***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) ***REMOVED***
***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies());
***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies());
***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
***REMOVED******REMOVED***
public String getDependencyFile(String language/*, String env*/)
***REMOVED***
return getDependencyFile(this.configuration,language);
***REMOVED***
private String getDependencyFile (Configuration configuration, String language)
***REMOVED***
this.logger.debug("Getting dependency file for language "+language);
switch (language)
***REMOVED***
case "R":
return configuration.getSVNCRANDepsList();
case "R-blackbox":
return configuration.getSVNRBDepsList();
case "Java":
return configuration.getSVNJavaDepsList();
case "Knime-Workflow":
return configuration.getSVNKWDepsList();
case "Knime-Workflow4.1":
return configuration.getSVNKW4_1DepsList();
case "Linux-compiled":
return configuration.getSVNLinuxCompiledDepsList();
case "Octave":
return configuration.getSVNOctaveDepsList();
case "Python":
return configuration.getSVNPythonDepsList();
case "Python3.6":
return configuration.getSVNPython3_6DepsList();
case "Pre-Installed":
return configuration.getSVNPreInstalledDepsList();
case "Windows-compiled":
return configuration.getSVNWCDepsList();
default:
***REMOVED***
***REMOVED***
***REMOVED***
public boolean updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/)
***REMOVED***
return this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
***REMOVED***
public void updateAlgorithmFiles(File a) throws SVNException, SVNCommitException***REMOVED***
***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a);
this.updateAlgorithmList(this.configuration.getRepository(), a);
***REMOVED***
private void updateAlgorithmList(String svnMainAlgoRepo, File algorithmsFile) throws SVNException, SVNCommitException
***REMOVED***
this.logger.debug("Adding .jar file: " + algorithmsFile + " to repository " + svnMainAlgoRepo);
try
***REMOVED***
if (fileExists(svnMainAlgoRepo+File.separator+algorithmsFile.getName(), -1))
***REMOVED***
this.updateFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo, algorithmsFile.getName());
***REMOVED***
else this.putFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo,algorithmsFile.getName());
***REMOVED***
catch (FileNotFoundException e)
***REMOVED***
this.logger.error("Temporary algorithm file not found: this exception should not happen",e);
***REMOVED***
finally
***REMOVED***
this.svnRepository.closeSession();
***REMOVED***
***REMOVED***
public void putFile(FileInputStream fileInputSreeam, String destinationFolder, String fileName) throws SVNException, SVNCommitException
***REMOVED***
this.logger.debug("Putting new file on the SVN repository");
final ISVNEditor commitEditor = svnRepository.getCommitEditor("Add algorithm to list", null);
commitEditor.openRoot(-1);
commitEditor.openDir(destinationFolder, -1);
String filePath = destinationFolder + "/" + fileName;
commitEditor.addFile(filePath, null, -1);
commitEditor.applyTextDelta(filePath, null);
SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
String checksum = deltaGenerator.sendDelta(filePath, fileInputSreeam, commitEditor, true);
commitEditor.closeFile(filePath, checksum);
commitEditor.closeDir();
commitEditor.closeDir();
SVNCommitInfo info = commitEditor.closeEdit();
SVNErrorMessage errorMessage = info.getErrorMessage();
if (errorMessage != null)
***REMOVED***
this.logger.error("Operation failed: "+errorMessage.getFullMessage());
throw new SVNCommitException(errorMessage,fileName);
***REMOVED***
this.logger.debug("Operation completed");
***REMOVED***
public void updateFile(FileInputStream fileInputStream, String destinationFolder, String fileName) throws SVNException, SVNCommitException ***REMOVED***
this.logger.debug("Updating existing file on the SVN repository");
final ISVNEditor commitEditor = svnRepository.getCommitEditor("Updating algorithm", null);
commitEditor.openRoot(-1);
commitEditor.openDir(destinationFolder, -1);
String filePath = destinationFolder + "/" + fileName;
***REMOVED*** if (fileExists(filePath, -1)) ***REMOVED*** ***REMOVED*** updating existing file
commitEditor.openFile(filePath, -1);
***REMOVED******REMOVED*** else ***REMOVED*** ***REMOVED*** creating new file
***REMOVED***commitEditor.addFile(filePath, null, -1);
***REMOVED******REMOVED***
commitEditor.applyTextDelta(filePath, null);
SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
String checksum = deltaGenerator.sendDelta(filePath, fileInputStream, commitEditor, true);
commitEditor.closeFile(filePath, checksum);
commitEditor.closeDir();
commitEditor.closeDir();
SVNCommitInfo info = commitEditor.closeEdit();
SVNErrorMessage errorMessage = info.getErrorMessage();
if (errorMessage != null)
***REMOVED***
this.logger.error("Operation failed: "+errorMessage.getFullMessage());
throw new SVNCommitException(errorMessage,fileName+" to be updated");
***REMOVED***
this.logger.debug("Operation completed");
***REMOVED***
public boolean fileExists(String path, long revision) throws SVNException ***REMOVED***
SVNNodeKind kind = svnRepository.checkPath(path, revision);
if (kind == SVNNodeKind.FILE) ***REMOVED***
return true;
***REMOVED***
return false;
***REMOVED***
***REMOVED*** public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** return new ByteArrayInputStream(FileUtils.readFileToByteArray(file));
***REMOVED******REMOVED***
private boolean updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/)
***REMOVED***
boolean response = false;
try ***REMOVED***
this.logger.debug("Updating algorithm list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> newContent = new LinkedList<>(Arrays.asList(lines));
***REMOVED*** check if the algorithm is already in the list (match the class name) and delete the content
for (String l : lines) ***REMOVED***
if (l.contains(algorithm.getClazz())) ***REMOVED***
newContent.remove(l);
***REMOVED***System.out.println("Not updating algorithm list beacuse already present");
***REMOVED***return;
***REMOVED***
***REMOVED***
***REMOVED*** the algorithm is not in the list or must be overwritten cause some modification. Add it
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, category,algorithm_type/*, env*/));
***REMOVED*** Collections.sort(newContent);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
byte[] originalContents = byteArrayOutputStream.toByteArray();
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update algorithm list", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : newContent) ***REMOVED***
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file, md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
SVNCommitInfo info = commitEditor.closeEdit();
SVNErrorMessage errorMessage = info.getErrorMessage();
if (errorMessage != null)
***REMOVED***
this.logger.error("Operation failed: "+errorMessage.getFullMessage());
response = false;
***REMOVED***
else response = true;
***REMOVED***
catch (Exception ex)
***REMOVED***
this.logger.error("Unable to commit algorithm list",ex);
response = false;
***REMOVED***
finally
***REMOVED***
svnRepository.closeSession();
***REMOVED***
return response;
***REMOVED***
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException ***REMOVED***
***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis());
***REMOVED***long unixTime = System.currentTimeMillis() / 1000L;
StringBuffer sb = new StringBuffer("| ");
sb.append(algorithm.getName() + " | ");
sb.append(algorithm.getFullname() + " | ");
sb.append(category + " | ");
sb.append("DataMinerPoolManager | ");
sb.append("<notextile>./addAlgorithm.sh " + algorithm.getName() + " " + algorithm.getCategory() + " "
+ algorithm.getClazz() + " " + targetVRE + " " + algorithm_type + " N "
+ algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
sb.append("none | ");
sb.append(this.getTimeZone() + " | ");
this.logger.info("Algo details: "+sb.toString());
return sb.toString();
***REMOVED***
public Collection<String> getUndefinedDependencies(String file, Collection<Dependency> deps) throws GenericException
***REMOVED***
try
***REMOVED***
***REMOVED*** SendMail sm = new SendMail();
***REMOVED*** NotificationHelper nh = new NotificationHelper();
List<String> undefined = new LinkedList<String>();
***REMOVED***to fix in next release: if the file is not present for that language in the service.properties then skip and return null list of string
***REMOVED***just to uncomment the following lines
if(file.isEmpty())***REMOVED***
return undefined;
***REMOVED***
this.logger.debug("Checking dependencies list: " + file);
List<String> validDependencies = new LinkedList<String>();
for (String singlefile: CheckMethod.getFiles(file))***REMOVED***
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(singlefile, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
for(String l: byteArrayOutputStream.toString().split("\\r?\\n"))***REMOVED***
validDependencies.add(l.trim());
***REMOVED******REMOVED***
this.logger.debug("Valid dependencies are: "+validDependencies);
for(Dependency d: deps)***REMOVED***
String depName = d.getName();
if(!validDependencies.contains(depName))***REMOVED***
undefined.add(depName);
***REMOVED***
***REMOVED***
return undefined;
***REMOVED***
***REMOVED***
***REMOVED*** for (String a : lines) ***REMOVED***
***REMOVED*** for (String b : ldep) ***REMOVED***
***REMOVED*** if (b.equals(a)) ***REMOVED***
***REMOVED*** System.out.println("The following dependency is correctly written: " + b);
***REMOVED*** ***REMOVED*** else
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** boolean check = false;
***REMOVED*** try ***REMOVED***
***REMOVED*** System.out.println("Checking dependencies list: " + file);
***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
***REMOVED*** svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
***REMOVED***
***REMOVED*** ***REMOVED*** if(deps.isEmpty())***REMOVED***
***REMOVED*** ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody());
***REMOVED*** ***REMOVED*** Exception e = new Exception("No dependency specified for this
***REMOVED*** ***REMOVED*** algorithm");
***REMOVED*** ***REMOVED*** throw e;
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) ***REMOVED***
***REMOVED*** List<String> ldep = new LinkedList<>();
***REMOVED*** for (Dependency d : deps) ***REMOVED***
***REMOVED*** ldep.add(d.getName());
***REMOVED*** ***REMOVED***
***REMOVED*** for (String a : lines) ***REMOVED***
***REMOVED*** for (String b : ldep) ***REMOVED***
***REMOVED*** if (b.equals(a)) ***REMOVED***
***REMOVED*** System.out.println("The following dependency is correctly written: " + b);
***REMOVED*** check = true;
***REMOVED*** ***REMOVED*** else
***REMOVED*** check = false;
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED*** catch (Exception a) ***REMOVED***
***REMOVED*** a.getMessage();
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** return check;
***REMOVED*** catch (SVNException e)
***REMOVED***
throw new GenericException(e);
***REMOVED***
***REMOVED***
public boolean checkIfAvaialable(String file, Collection<Dependency> deps) throws SVNException ***REMOVED***
***REMOVED***SendMail sm = new SendMail();
***REMOVED***NotificationHelper nh = new NotificationHelper();
boolean check = false;
try ***REMOVED***
this.logger.info("Checking dependencies list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
***REMOVED*** if(deps.isEmpty())***REMOVED***
***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody());
***REMOVED*** Exception e = new Exception("No dependency specified for this
***REMOVED*** algorithm");
***REMOVED*** throw e;
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** else if (!deps.isEmpty()) ***REMOVED***
List<String> ldep = new LinkedList<>();
for (Dependency d : deps) ***REMOVED***
ldep.add(d.getName());
***REMOVED***
for (String a : lines) ***REMOVED***
for (String b : ldep) ***REMOVED***
if (b.equals(a)) ***REMOVED***
System.out.println("The following dependency is correctly written: " + b);
check = true;
***REMOVED*** else
check = false;
***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** catch (Exception a)
***REMOVED***
this.logger.error(a.getMessage(),a);
***REMOVED***
return check;
***REMOVED***
public void updateSVN(String file, Collection<Dependency> deps) ***REMOVED***
try ***REMOVED***
this.logger.info("Updating dependencies list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> ldep = new LinkedList<>();
for (Dependency d : deps) ***REMOVED***
ldep.add(d.getName());
***REMOVED***
List<String> aa = this.checkMatch(lines, ldep);
Collections.sort(aa);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
byte[] originalContents = byteArrayOutputStream.toByteArray();
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : aa) ***REMOVED***
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file, md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
commitEditor.closeEdit();
***REMOVED*** catch (Exception ex) ***REMOVED***
ex.printStackTrace();
***REMOVED***
finally ***REMOVED***
svnRepository.closeSession();
***REMOVED***
***REMOVED***
public static String md5(byte[] contents) ***REMOVED***
final byte[] tmp = new byte[1024];
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents),
"md5");
try ***REMOVED***
while (checksumStream.read(tmp) > 0) ***REMOVED***
***REMOVED***
***REMOVED***
return checksumStream.getDigest();
***REMOVED*** catch (IOException e) ***REMOVED***
***REMOVED*** never happens
e.printStackTrace();
***REMOVED***
***REMOVED*** finally ***REMOVED***
SVNFileUtil.closeFile(checksumStream);
***REMOVED***
***REMOVED***
public List<String> checkMatch(String[] lines, List<String> ls) ***REMOVED***
Set<String> ss = new HashSet<String>(ls);
ss.addAll(Arrays.asList(lines));
return new ArrayList<>(ss);
***REMOVED***
public String getTimeZone() throws ParseException***REMOVED***
Calendar cal = Calendar.getInstance();
cal.getTime();
DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy");
Date fromDate = formatter.parse(cal.getTime().toString());
TimeZone central = TimeZone.getTimeZone("UTC");
formatter.setTimeZone(central);
this.logger.info(formatter.format(fromDate));
return formatter.format(fromDate);
***REMOVED***
public static void main(String[] args) throws SVNException, ParseException ***REMOVED***
***REMOVED*** SVNUpdater c = new SVNUpdater();
***REMOVED***File a = new File("/home/ngalante/Desktop/testCiro");
***REMOVED***File b = new File ("/home/ngalante/Desktop/testB");
***REMOVED***long unixTime = System.currentTimeMillis() / 1000L;
***REMOVED***System.out.println(unixTime);
***REMOVED***c.updateAlgorithmFiles(a);
***REMOVED***c.updateAlgorithmFiles(b);
***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis());
Calendar cal = Calendar.getInstance();
cal.getTime();
DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy");
Date fromDate = formatter.parse(cal.getTime().toString());
TimeZone central = TimeZone.getTimeZone("UTC");
formatter.setTimeZone(central);
System.out.println(formatter.format(fromDate));
***REMOVED***
***REMOVED***