package org.gcube.dataanalysis.dataminer.poolmanager.util; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; ***REMOVED*** import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.HashSet; ***REMOVED*** ***REMOVED*** import java.util.Set; import java.util.TimeZone; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; ***REMOVED*** ***REMOVED*** import org.tmatesoft.svn.core.SVNCommitInfo; import org.tmatesoft.svn.core.SVNErrorMessage; import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNNodeKind; import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; import org.tmatesoft.svn.core.internal.wc.admin.SVNChecksumInputStream; import org.tmatesoft.svn.core.io.ISVNEditor; import org.tmatesoft.svn.core.io.SVNRepository; import org.tmatesoft.svn.core.io.diff.SVNDeltaGenerator; /** * Created by ggiammat on 5/9/17. */ public abstract class SVNUpdater { private SVNRepository svnRepository; private Configuration configuration; ***REMOVED*** public SVNUpdater(Configuration configuration) throws SVNException { this.configuration = configuration; this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository(); this.logger = LoggerFactory.getLogger(SVNUpdater.class); ***REMOVED*** ***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) { ***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); ***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); ***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); ***REMOVED******REMOVED*** public String getDependencyFile(String language/*, String env*/) { return getDependencyFile(this.configuration,language); ***REMOVED*** private String getDependencyFile (Configuration configuration, String language) { this.logger.debug("Getting dependency file for language "+language); switch (language) { case "R": return configuration.getSVNCRANDepsList(); case "R-blackbox": return configuration.getSVNRBDepsList(); case "Java": return configuration.getSVNJavaDepsList(); case "Knime-Workflow": return configuration.getSVNKWDepsList(); case "Knime-Workflow4.1": return configuration.getSVNKW4_1DepsList(); case "Linux-compiled": return configuration.getSVNLinuxCompiledDepsList(); case "Octave": return configuration.getSVNOctaveDepsList(); case "Python": return configuration.getSVNPythonDepsList(); case "Python3.6": return configuration.getSVNPython3_6DepsList(); case "Pre-Installed": return configuration.getSVNPreInstalledDepsList(); case "Windows-compiled": return configuration.getSVNWCDepsList(); default: return null; ***REMOVED*** ***REMOVED*** public boolean updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) { return this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); ***REMOVED*** public void updateAlgorithmFiles(File a) throws SVNException, SVNCommitException{ ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); this.updateAlgorithmList(this.configuration.getRepository(), a); ***REMOVED*** private void updateAlgorithmList(String svnMainAlgoRepo, File algorithmsFile) throws SVNException, SVNCommitException { this.logger.debug("Adding .jar file: " + algorithmsFile + " to repository " + svnMainAlgoRepo); try { if (fileExists(svnMainAlgoRepo+File.separator+algorithmsFile.getName(), -1)) { this.updateFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo, algorithmsFile.getName()); ***REMOVED*** else this.putFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo,algorithmsFile.getName()); ***REMOVED*** catch (FileNotFoundException e) { this.logger.error("Temporary algorithm file not found: this exception should not happen",e); ***REMOVED*** finally { this.svnRepository.closeSession(); ***REMOVED*** ***REMOVED*** public void putFile(FileInputStream fileInputSreeam, String destinationFolder, String fileName) throws SVNException, SVNCommitException { this.logger.debug("Putting new file on the SVN repository"); final ISVNEditor commitEditor = svnRepository.getCommitEditor("Add algorithm to list", null); commitEditor.openRoot(-1); commitEditor.openDir(destinationFolder, -1); String filePath = destinationFolder + "/" + fileName; commitEditor.addFile(filePath, null, -1); commitEditor.applyTextDelta(filePath, null); SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); String checksum = deltaGenerator.sendDelta(filePath, fileInputSreeam, commitEditor, true); commitEditor.closeFile(filePath, checksum); commitEditor.closeDir(); commitEditor.closeDir(); SVNCommitInfo info = commitEditor.closeEdit(); SVNErrorMessage errorMessage = info.getErrorMessage(); if (errorMessage != null) { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); throw new SVNCommitException(errorMessage,fileName); ***REMOVED*** this.logger.debug("Operation completed"); ***REMOVED*** public void updateFile(FileInputStream fileInputStream, String destinationFolder, String fileName) throws SVNException, SVNCommitException { this.logger.debug("Updating existing file on the SVN repository"); final ISVNEditor commitEditor = svnRepository.getCommitEditor("Updating algorithm", null); commitEditor.openRoot(-1); commitEditor.openDir(destinationFolder, -1); String filePath = destinationFolder + "/" + fileName; ***REMOVED*** if (fileExists(filePath, -1)) { ***REMOVED*** updating existing file commitEditor.openFile(filePath, -1); ***REMOVED******REMOVED*** else { ***REMOVED*** creating new file ***REMOVED***commitEditor.addFile(filePath, null, -1); ***REMOVED******REMOVED*** commitEditor.applyTextDelta(filePath, null); SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); String checksum = deltaGenerator.sendDelta(filePath, fileInputStream, commitEditor, true); commitEditor.closeFile(filePath, checksum); commitEditor.closeDir(); commitEditor.closeDir(); SVNCommitInfo info = commitEditor.closeEdit(); SVNErrorMessage errorMessage = info.getErrorMessage(); if (errorMessage != null) { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); throw new SVNCommitException(errorMessage,fileName+" to be updated"); ***REMOVED*** this.logger.debug("Operation completed"); ***REMOVED*** public boolean fileExists(String path, long revision) throws SVNException { SVNNodeKind kind = svnRepository.checkPath(path, revision); if (kind == SVNNodeKind.FILE) { return true; ***REMOVED*** return false; ***REMOVED*** ***REMOVED*** public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException ***REMOVED*** { ***REMOVED*** ***REMOVED*** return new ByteArrayInputStream(FileUtils.readFileToByteArray(file)); ***REMOVED******REMOVED*** private boolean updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) { boolean response = false; try { this.logger.debug("Updating algorithm list: " + file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); List newContent = new LinkedList<>(Arrays.asList(lines)); ***REMOVED*** check if the algorithm is already in the list (match the class name) and delete the content for (String l : lines) { if (l.contains(algorithm.getClazz())) { newContent.remove(l); ***REMOVED***System.out.println("Not updating algorithm list beacuse already present"); ***REMOVED***return; ***REMOVED*** ***REMOVED*** ***REMOVED*** the algorithm is not in the list or must be overwritten cause some modification. Add it newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, category,algorithm_type/*, env*/)); ***REMOVED*** Collections.sort(newContent); final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); byte[] originalContents = byteArrayOutputStream.toByteArray(); final ISVNEditor commitEditor = svnRepository.getCommitEditor("update algorithm list", null); commitEditor.openRoot(-1); commitEditor.openFile(file, -1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); for (String line : newContent) { baos.write(line.getBytes()); baos.write("\n".getBytes()); ***REMOVED*** byte[] bytes = baos.toByteArray(); commitEditor.applyTextDelta(file, md5(originalContents)); final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, new ByteArrayInputStream(bytes), commitEditor, true); commitEditor.closeFile(file, checksum); SVNCommitInfo info = commitEditor.closeEdit(); SVNErrorMessage errorMessage = info.getErrorMessage(); if (errorMessage != null) { this.logger.error("Operation failed: "+errorMessage.getFullMessage()); response = false; ***REMOVED*** else response = true; ***REMOVED*** catch (Exception ex) { this.logger.error("Unable to commit algorithm list",ex); response = false; ***REMOVED*** finally { svnRepository.closeSession(); ***REMOVED*** return response; ***REMOVED*** public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException { ***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis()); ***REMOVED***long unixTime = System.currentTimeMillis() / 1000L; StringBuffer sb = new StringBuffer("| "); sb.append(algorithm.getName() + " | "); sb.append(algorithm.getFullname() + " | "); sb.append(category + " | "); sb.append("DataMinerPoolManager | "); sb.append("./addAlgorithm.sh " + algorithm.getName() + " " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm_type + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); sb.append("none | "); sb.append(this.getTimeZone() + " | "); this.logger.info("Algo details: "+sb.toString()); return sb.toString(); ***REMOVED*** public Collection getUndefinedDependencies(String file, Collection deps) throws GenericException { try { ***REMOVED*** SendMail sm = new SendMail(); ***REMOVED*** NotificationHelper nh = new NotificationHelper(); List undefined = new LinkedList(); ***REMOVED***to fix in next release: if the file is not present for that language in the service.properties then skip and return null list of string ***REMOVED***just to uncomment the following lines if(file.isEmpty()){ return undefined; ***REMOVED*** this.logger.debug("Checking dependencies list: " + file); List validDependencies = new LinkedList(); for (String singlefile: CheckMethod.getFiles(file)){ final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(singlefile, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); for(String l: byteArrayOutputStream.toString().split("\\r?\\n")){ validDependencies.add(l.trim()); ***REMOVED******REMOVED*** this.logger.debug("Valid dependencies are: "+validDependencies); for(Dependency d: deps){ String depName = d.getName(); if(!validDependencies.contains(depName)){ undefined.add(depName); ***REMOVED*** ***REMOVED*** return undefined; ***REMOVED*** ***REMOVED*** ***REMOVED*** for (String a : lines) { ***REMOVED*** for (String b : ldep) { ***REMOVED*** if (b.equals(a)) { ***REMOVED*** System.out.println("The following dependency is correctly written: " + b); ***REMOVED*** ***REMOVED*** else ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** boolean check = false; ***REMOVED*** try { ***REMOVED*** System.out.println("Checking dependencies list: " + file); ***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); ***REMOVED*** svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); ***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); ***REMOVED*** ***REMOVED*** ***REMOVED*** if(deps.isEmpty()){ ***REMOVED*** ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); ***REMOVED*** ***REMOVED*** Exception e = new Exception("No dependency specified for this ***REMOVED*** ***REMOVED*** algorithm"); ***REMOVED*** ***REMOVED*** throw e; ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) { ***REMOVED*** List ldep = new LinkedList<>(); ***REMOVED*** for (Dependency d : deps) { ***REMOVED*** ldep.add(d.getName()); ***REMOVED*** ***REMOVED*** ***REMOVED*** for (String a : lines) { ***REMOVED*** for (String b : ldep) { ***REMOVED*** if (b.equals(a)) { ***REMOVED*** System.out.println("The following dependency is correctly written: " + b); ***REMOVED*** check = true; ***REMOVED*** ***REMOVED*** else ***REMOVED*** check = false; ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** catch (Exception a) { ***REMOVED*** a.getMessage(); ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** return check; ***REMOVED*** catch (SVNException e) { throw new GenericException(e); ***REMOVED*** ***REMOVED*** public boolean checkIfAvaialable(String file, Collection deps) throws SVNException { ***REMOVED***SendMail sm = new SendMail(); ***REMOVED***NotificationHelper nh = new NotificationHelper(); boolean check = false; try { this.logger.info("Checking dependencies list: " + file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); ***REMOVED*** if(deps.isEmpty()){ ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); ***REMOVED*** Exception e = new Exception("No dependency specified for this ***REMOVED*** algorithm"); ***REMOVED*** throw e; ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) { List ldep = new LinkedList<>(); for (Dependency d : deps) { ldep.add(d.getName()); ***REMOVED*** for (String a : lines) { for (String b : ldep) { if (b.equals(a)) { System.out.println("The following dependency is correctly written: " + b); check = true; ***REMOVED*** else check = false; ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** catch (Exception a) { this.logger.error(a.getMessage(),a); ***REMOVED*** return check; ***REMOVED*** public void updateSVN(String file, Collection deps) { try { this.logger.info("Updating dependencies list: " + file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); List ldep = new LinkedList<>(); for (Dependency d : deps) { ldep.add(d.getName()); ***REMOVED*** List aa = this.checkMatch(lines, ldep); Collections.sort(aa); final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); byte[] originalContents = byteArrayOutputStream.toByteArray(); final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); commitEditor.openRoot(-1); commitEditor.openFile(file, -1); ByteArrayOutputStream baos = new ByteArrayOutputStream(); for (String line : aa) { baos.write(line.getBytes()); baos.write("\n".getBytes()); ***REMOVED*** byte[] bytes = baos.toByteArray(); commitEditor.applyTextDelta(file, md5(originalContents)); final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, new ByteArrayInputStream(bytes), commitEditor, true); commitEditor.closeFile(file, checksum); commitEditor.closeEdit(); ***REMOVED*** catch (Exception ex) { ex.printStackTrace(); ***REMOVED*** finally { svnRepository.closeSession(); ***REMOVED*** ***REMOVED*** public static String md5(byte[] contents) { final byte[] tmp = new byte[1024]; final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5"); try { while (checksumStream.read(tmp) > 0) { ***REMOVED*** ***REMOVED*** return checksumStream.getDigest(); ***REMOVED*** catch (IOException e) { ***REMOVED*** never happens e.printStackTrace(); return null; ***REMOVED*** finally { SVNFileUtil.closeFile(checksumStream); ***REMOVED*** ***REMOVED*** public List checkMatch(String[] lines, List ls) { Set ss = new HashSet(ls); ss.addAll(Arrays.asList(lines)); return new ArrayList<>(ss); ***REMOVED*** public String getTimeZone() throws ParseException{ Calendar cal = Calendar.getInstance(); cal.getTime(); DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy"); Date fromDate = formatter.parse(cal.getTime().toString()); TimeZone central = TimeZone.getTimeZone("UTC"); formatter.setTimeZone(central); this.logger.info(formatter.format(fromDate)); return formatter.format(fromDate); ***REMOVED*** public static void main(String[] args) throws SVNException, ParseException { ***REMOVED*** SVNUpdater c = new SVNUpdater(); ***REMOVED***File a = new File("/home/ngalante/Desktop/testCiro"); ***REMOVED***File b = new File ("/home/ngalante/Desktop/testB"); ***REMOVED***long unixTime = System.currentTimeMillis() / 1000L; ***REMOVED***System.out.println(unixTime); ***REMOVED***c.updateAlgorithmFiles(a); ***REMOVED***c.updateAlgorithmFiles(b); ***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis()); Calendar cal = Calendar.getInstance(); cal.getTime(); DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy"); Date fromDate = formatter.parse(cal.getTime().toString()); TimeZone central = TimeZone.getTimeZone("UTC"); formatter.setTimeZone(central); System.out.println(formatter.format(fromDate)); ***REMOVED*** ***REMOVED***