From 85b523e2f20a1069206403c27aab6c6dbbab6263 Mon Sep 17 00:00:00 2001 From: Ciro Formisano Date: Fri, 15 Dec 2017 12:01:42 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@160557 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 2 +- .../DMPMClientConfiguratorManager.java | 40 +++- .../poolmanager/service/DMPMJob.java | 38 +++- .../poolmanager/util/CheckMethod.java | 177 +++++++++++++-- .../poolmanager/util/NotificationHelper.java | 17 +- .../poolmanager/util/SVNUpdater.java | 183 ++++++++++------ .../dataminer/poolmanager/util/SendMail.java | 76 ++++--- .../util/exception/AlgorithmException.java | 26 +++ .../util/exception/SVNCommitException.java | 41 ++++ .../util/impl/CheckMethodProduction.java | 199 ++--------------- .../util/impl/CheckMethodStaging.java | 205 ++---------------- .../impl/NotificationHelperProduction.java | 17 +- .../util/impl/NotificationHelperStaging.java | 22 +- src/main/resources/default.admins | 6 + 14 files changed, 512 insertions(+), 537 deletions(-) create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java create mode 100755 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java create mode 100755 src/main/resources/default.admins diff --git a/pom.xml b/pom.xml index 8a14623..f438ea0 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ org.gcube.dataAnalysis dataminer-pool-manager war - 2.3.0-SNAPSHOT + 2.4.0-SNAPSHOT dataminer-pool-manager diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java index 439c580..1ad903e 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java @@ -1,6 +1,8 @@ package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration; ***REMOVED*** +import java.util.Iterator; +import java.util.Properties; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.clients.ScopedCacheMap; @@ -11,15 +13,28 @@ import org.tmatesoft.svn.core.SVNException; public class DMPMClientConfiguratorManager{ private final Logger logger; - + private Properties defaultAdmins; + private String admins; + static DMPMClientConfiguratorManager instance; private ScopedCacheMap cacheMap; private DMPMClientConfiguratorManager () { + this.admins = null; this.cacheMap = new ScopedCacheMap(); this.logger = LoggerFactory.getLogger(DMPMClientConfiguratorManager.class); + this.defaultAdmins = new Properties(); + + try + { + this.defaultAdmins.load(this.getClass().getResourceAsStream("/default.admins")); + this.logger.debug("Default users successfully loaded"); + ***REMOVED*** catch (Exception e) + { + this.logger.error("Unable to get default users",e); + ***REMOVED*** ***REMOVED*** private ClientConfigurationCache getCurrentCache () @@ -60,6 +75,29 @@ public class DMPMClientConfiguratorManager{ return new ConfigurationImpl(CONFIGURATIONS.STAGE, getCurrentCache()); ***REMOVED*** + public String getDefaultAdmins () + { + if (this.admins == null && this.defaultAdmins.isEmpty()) this.admins= "ciro.formisano"; + else if (admins == null) + { + Iterator keys = this.defaultAdmins.keySet().iterator(); + StringBuilder response = new StringBuilder(); + + while (keys.hasNext()) + { + String key = (String) keys.next(); + response.append(this.defaultAdmins.getProperty(key)).append(", "); + ***REMOVED*** + + this.admins = response.substring(0, response.length()-2); + ***REMOVED*** + + this.logger.debug("Default admins list "+this.admins); + return this.admins; + +***REMOVED*** + + public static void main(String[] args) throws IOException, SVNException { DMPMClientConfiguratorManager a = new DMPMClientConfiguratorManager(); ScopeProvider.instance.set("/gcube/devNext/NextNext"); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java index 8d841af..cd33818 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java @@ -19,7 +19,9 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; +import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.AlgorithmException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException; +import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; ***REMOVED*** ***REMOVED*** import org.tmatesoft.svn.core.SVNException; @@ -186,18 +188,34 @@ public abstract class DMPMJob { ***REMOVED***this.setStatusInformation(STATUS.PROGRESS); this.logger.debug("Checking the method..."); - if (methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get())&&(methodChecker.algoExists(this.algorithm))) - { + try + { + methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get()); + methodChecker.copyAlgorithms(this.algorithm); this.logger.debug("Method OK and algo exists"); - System.out.println("Interface check ok!"); - System.out.println("Both the files exist at the correct path!"); - - this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.vREName,this.category, this.algorithm_type, + this.logger.debug("Interface check ok!"); + this.logger.debug("Both the files exist at the correct path!"); + boolean algorithmListResult = this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.vREName,this.category, this.algorithm_type, this.algorithm.getFullname()); this.setStatusInformation(STATUS.COMPLETED); - sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo())); - return; - ***REMOVED*** else + + String bodyResponse = algorithmListResult ? nh.getSuccessBody("\n\n"+this.buildInfo()) : nh.getSuccessBody("\n\n"+this.buildInfo())+"\nWARNING: algorithm list could not be updated on SVN"; + + sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", bodyResponse); + + ***REMOVED*** catch (SVNCommitException e) + { + this.logger.error("Unable to complete the commit operation: "+e.getMessage()); + this.setStatusInformation(STATUS.FAILED); + sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm", + nh.getFailedBody( + "\n"+ + "Commit operation failed for "+e.getFileName() + + "the message of the SVN Server is the following:\n"+e.getSvnErrorMessage().getMessage()+"\n\n" + +this.buildInfo())); + + + ***REMOVED*** catch (AlgorithmException ae) { this.logger.debug("Operation failed"); this.setStatusInformation(STATUS.FAILED); @@ -207,8 +225,8 @@ public abstract class DMPMJob { "Installation completed but DataMiner Interface not working correctly or files " + this.algorithm.getName() + ".jar and " + this.algorithm.getName() + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo()); - ***REMOVED*** + ***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java index 173a971..0720277 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java @@ -9,28 +9,50 @@ import java.util.ArrayList; import java.util.Arrays; ***REMOVED*** ***REMOVED*** +import java.util.Properties; +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; +import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.AlgorithmException; +import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; ***REMOVED*** ***REMOVED*** +import org.tmatesoft.svn.core.SVNException; + +import com.jcraft.jsch.Channel; +import com.jcraft.jsch.ChannelSftp; +import com.jcraft.jsch.JSch; +import com.jcraft.jsch.JSchException; +import com.jcraft.jsch.Session; +import com.jcraft.jsch.SftpException; public abstract class CheckMethod { ***REMOVED*** - - public CheckMethod() + private Configuration configuration; + + private final String KNOWN_HOSTS= "~/.ssh/known_hosts", + PRIVATE_KEY = "~/.ssh/id_rsa", + SSH_USER = "root", + SFTP_PROTOCOL = "sftp", + TEMP_DIRECTORY = "tmp"; + private final Properties sshConfig; + + public CheckMethod(Configuration configuration) { this.logger = LoggerFactory.getLogger(CheckMethod.class); + sshConfig = new java.util.Properties(); + sshConfig.put("StrictHostKeyChecking", "no"); ***REMOVED*** - public boolean checkMethod(String machine, String token) throws Exception { + public void checkMethod(String machine, String token) throws AlgorithmException { try { this.logger.debug("Checking method for machine "+machine); this.logger.debug("By using tocken "+token); - System.out.println("Machine: " + machine); + this.logger.debug("Machine: " + machine); ***REMOVED*** String getCapabilitesRequest = new String(); ***REMOVED*** String getCapabilitesResponse = new String(); - System.out.println(" Token: " + token); + this.logger.debug(" Token: " + token); String request = "http:***REMOVED***" + machine + "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token; String response = machine + "___" + token + ".xml"; @@ -72,12 +94,12 @@ public abstract class CheckMethod { String operatorDescription = innerLine.substring(innerLine.indexOf(">") + 1); operatorDescription = operatorDescription.substring(0, operatorDescription.indexOf("<")); this.logger.debug("Operator descriptor "+operatorDescription); - System.out.println(" " + operatorDescription); + this.logger.debug(" " + operatorDescription); innerFlag = false; ***REMOVED*** else if (innerLine.contains("ows:ExceptionText")) { this.logger.debug("Exception found"); - System.out.println(" " + "error retrieving operator description"); + this.logger.debug(" " + "error retrieving operator description"); innerFlag = false; flag = false; ***REMOVED*** else @@ -91,29 +113,152 @@ public abstract class CheckMethod { this.logger.debug("Operation successful"); fileWriter.close(); - return true; + ***REMOVED*** catch (Exception e) { - e.getMessage(); - this.logger.error("Error "+e.getMessage(),e); - return false; + + throw new AlgorithmException("Error "+e.getMessage(),e); + ***REMOVED*** ***REMOVED*** - - public abstract boolean algoExists(Algorithm algo/*, String env*/) throws Exception; + + public void copyAlgorithms(Algorithm algo/*, String env*/) throws SVNCommitException, Exception{ + + this.logger.debug("Looking if algo "+algo.getName()+ " exists"); + File file = new File(this.configuration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar"); + File file2 = new File(this.configuration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar"); + this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath()); + + + if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/))) + { + this.logger.debug("Files found"); + this.copyFromDmToSVN(file/*,env*/); + this.copyFromDmToSVN(file2/*,env*/); + this.logger.debug("Files have been copied to SVN"); + ***REMOVED*** + else + { + this.logger.debug("Files not found"); + this.logger.debug("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); + throw new AlgorithmException("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); + ***REMOVED*** + +***REMOVED*** + - public abstract void deleteFiles(Algorithm a/*,String env*/) throws Exception; + public void deleteFiles(Algorithm a/*,String env*/) throws Exception + { + Session session = generateSession(); + System.out.println("checking existing in env: " + this.configuration.getHost()); + + File file = new File(this.configuration.getGhostAlgoDirectory()+"/"+a.getName()+".jar"); + File file2 = new File(this.configuration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); + + + System.out.println("First file is located to: "+file.getPath()); + System.out.println("Second file is located to: "+file2.getPath()); + + + System.out.println("session created."); + session.setConfig(this.sshConfig); + session.connect(); + Channel channel = session.openChannel(SFTP_PROTOCOL); + channel.connect(); + System.out.println("shell channel connected...."); + + ChannelSftp c = (ChannelSftp) channel; + + if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){ + + c.rm(file.getPath()); + c.rm(file2.getPath()); + System.out.println("Both the files have been deleted"); + ***REMOVED*** + else System.out.println("Files not found"); + channel.disconnect(); + c.disconnect(); + session.disconnect(); + +***REMOVED*** - public abstract boolean doesExist(String path/*, String env*/) throws Exception; + + public boolean doesExist(String path/*, String env*/) throws Exception { + + Session session = generateSession(); + boolean success = false; + session.connect(); + Channel channel = session.openChannel(SFTP_PROTOCOL); + channel.connect(); + System.out.println("shell channel connected...."); + + ChannelSftp c = (ChannelSftp) channel; + + + System.out.println(path); + + try { + c.lstat(path); + success = true; + ***REMOVED*** catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + ***REMOVED*** file doesn't exist + success = false; + ***REMOVED*** + ***REMOVED***success = true; ***REMOVED*** something else went wrong + ***REMOVED*** + channel.disconnect(); + c.disconnect(); + session.disconnect(); + this.logger.debug("Operation result "+success); + return success; + +***REMOVED*** + - public abstract void copyFromDmToSVN(File a/*,String env*/) throws Exception; + protected abstract void copyFromDmToSVN(File a) throws SVNCommitException, Exception; + + + protected void copyFromDmToSVN(File algorithmsFile/*,String env*/,SVNUpdater svnUpdater) throws SVNException, SVNCommitException, JSchException, SftpException { + + this.logger.debug("Copying algorithm file from Data Miner to SVN"); + String fileName = algorithmsFile.getName(); + this.logger.debug("File name "+fileName); + Session session = generateSession(); + session.connect(); + Channel channel = session.openChannel(SFTP_PROTOCOL); + channel.connect(); + ChannelSftp sftp = (ChannelSftp) channel; + sftp.cd(this.configuration.getGhostAlgoDirectory()); + String remoteFile = new StringBuilder(this.configuration.getGhostAlgoDirectory()).append(File.separator).append(fileName).toString(); + this.logger.debug("Remote file "+remoteFile); + String localFile = new StringBuilder(File.separator).append(TEMP_DIRECTORY).append(File.separator).append(fileName).toString(); + this.logger.debug("Local file "+localFile); + sftp.get(remoteFile,localFile); + channel.disconnect(); + session.disconnect(); + File f = new File(localFile); + svnUpdater.updateAlgorithmFiles(f); + f.delete(); +***REMOVED*** + private Session generateSession () throws JSchException + { + JSch jsch = new JSch(); + jsch.setKnownHosts(KNOWN_HOSTS); + jsch.addIdentity(PRIVATE_KEY); + this.logger.debug("Private Key Added."); + Session session = jsch.getSession(SSH_USER, this.configuration.getHost()); + this.logger.debug("session created."); + session.setConfig(this.sshConfig); + return session; +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java index ff686c4..a929a40 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -4,6 +4,13 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; public abstract class NotificationHelper { + private String subjectHeader; + + protected NotificationHelper (String subjectHeader) + { + this.subjectHeader = subjectHeader; +***REMOVED*** + ***REMOVED*** private Exception executionException; @@ -17,10 +24,16 @@ public abstract class NotificationHelper { ***REMOVED*** this.executionException = executionException; ***REMOVED*** ***REMOVED*** - public abstract String getSuccessSubject(); + public String getSuccessSubject() { + return this.subjectHeader+" is SUCCESS"; + ***REMOVED*** - public abstract String getFailedSubject(); + + + public String getFailedSubject() { + return String.format(this.subjectHeader+" is FAILED"); + ***REMOVED*** public String getSuccessBody(String info) { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index b937af5..2e70884 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -3,6 +3,8 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; ***REMOVED*** import java.text.DateFormat; import java.text.ParseException; @@ -19,10 +21,14 @@ import java.util.HashSet; import java.util.Set; import java.util.TimeZone; -import org.apache.commons.io.FileUtils; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; +import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; +***REMOVED*** +***REMOVED*** +import org.tmatesoft.svn.core.SVNCommitInfo; +import org.tmatesoft.svn.core.SVNErrorMessage; import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNNodeKind; import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; @@ -38,11 +44,12 @@ public abstract class SVNUpdater { private SVNRepository svnRepository; private Configuration configuration; +***REMOVED*** public SVNUpdater(Configuration configuration) throws SVNException { this.configuration = configuration; this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository(); - + this.logger = LoggerFactory.getLogger(SVNUpdater.class); ***REMOVED*** ***REMOVED*** public void updateRPRotoDeps(Algorithm algorithm) { @@ -64,7 +71,10 @@ public abstract class SVNUpdater { private String getDependencyFile (Configuration configuration, String language) { - switch (language) { + this.logger.debug("Getting dependency file for language "+language); + + switch (language) + { case "R": return configuration.getSVNCRANDepsList(); case "R-blackbox": @@ -89,14 +99,14 @@ public abstract class SVNUpdater { ***REMOVED*** - public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) + public boolean updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) { - this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); + return this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); ***REMOVED*** - public void updateAlgorithmFiles(File a) throws SVNException{ + public void updateAlgorithmFiles(File a) throws SVNException, SVNCommitException{ ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); this.updateAlgorithmList(this.configuration.getRepository(), a); ***REMOVED*** @@ -106,82 +116,92 @@ public abstract class SVNUpdater { - private void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException { - try { - System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo); - - if (fileExists(svnMainAlgoRepo+File.separator+a.getName(), -1)){ - this.updateFile(reteriveByteArrayInputStream(a), svnMainAlgoRepo, a.getName()); + private void updateAlgorithmList(String svnMainAlgoRepo, File algorithmsFile) throws SVNException, SVNCommitException + { + this.logger.debug("Adding .jar file: " + algorithmsFile + " to repository " + svnMainAlgoRepo); + + try + { + + if (fileExists(svnMainAlgoRepo+File.separator+algorithmsFile.getName(), -1)) + { + this.updateFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo, algorithmsFile.getName()); ***REMOVED*** - else this.putFile(reteriveByteArrayInputStream(a), svnMainAlgoRepo,a.getName()); - - ***REMOVED*** catch (Exception ex) { - ex.printStackTrace(); + else this.putFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo,algorithmsFile.getName()); + ***REMOVED*** + catch (FileNotFoundException e) + { + this.logger.error("Temporary algorithm file not found: this exception should not happen",e); ***REMOVED*** - - finally { - svnRepository.closeSession(); + finally + { + this.svnRepository.closeSession(); ***REMOVED*** ***REMOVED*** - public void putFile(ByteArrayInputStream byteArrayInputStream, String destinationFolder, String fileName) - throws SVNException { - - + public void putFile(FileInputStream fileInputSreeam, String destinationFolder, String fileName) throws SVNException, SVNCommitException + { + this.logger.debug("Putting new file on the SVN repository"); final ISVNEditor commitEditor = svnRepository.getCommitEditor("Add algorithm to list", null); commitEditor.openRoot(-1); commitEditor.openDir(destinationFolder, -1); + String filePath = destinationFolder + "/" + fileName; + commitEditor.addFile(filePath, null, -1); + commitEditor.applyTextDelta(filePath, null); + SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + String checksum = deltaGenerator.sendDelta(filePath, fileInputSreeam, commitEditor, true); + commitEditor.closeFile(filePath, checksum); + commitEditor.closeDir(); + commitEditor.closeDir(); + SVNCommitInfo info = commitEditor.closeEdit(); + SVNErrorMessage errorMessage = info.getErrorMessage(); - - String filePath = destinationFolder + "/" + fileName; - - - ***REMOVED***commitEditor.openFile(filePath, -1); - - - - commitEditor.addFile(filePath, null, -1); - - commitEditor.applyTextDelta(filePath, null); - SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); - String checksum = deltaGenerator.sendDelta(filePath, byteArrayInputStream, commitEditor, true); - commitEditor.closeFile(filePath, checksum); - commitEditor.closeDir(); - commitEditor.closeDir(); - commitEditor.closeEdit(); + if (errorMessage != null) + { + this.logger.error("Operation failed: "+errorMessage.getFullMessage()); + SVNCommitException exception = new SVNCommitException(errorMessage); + exception.setFileName(fileName); + throw exception; ***REMOVED*** + + this.logger.debug("Operation completed"); +***REMOVED*** - public void updateFile(ByteArrayInputStream byteArrayInputStream, String destinationFolder, String fileName) - throws SVNException { + public void updateFile(FileInputStream fileInputStream, String destinationFolder, String fileName) throws SVNException, SVNCommitException { - - final ISVNEditor commitEditor = svnRepository.getCommitEditor("Add algorithm to list", null); + this.logger.debug("Updating existing file on the SVN repository"); + final ISVNEditor commitEditor = svnRepository.getCommitEditor("Updating algorithm", null); commitEditor.openRoot(-1); commitEditor.openDir(destinationFolder, -1); + String filePath = destinationFolder + "/" + fileName; + ***REMOVED*** if (fileExists(filePath, -1)) { ***REMOVED*** updating existing file + commitEditor.openFile(filePath, -1); + ***REMOVED******REMOVED*** else { ***REMOVED*** creating new file + ***REMOVED***commitEditor.addFile(filePath, null, -1); + ***REMOVED******REMOVED*** + commitEditor.applyTextDelta(filePath, null); + SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + String checksum = deltaGenerator.sendDelta(filePath, fileInputStream, commitEditor, true); + commitEditor.closeFile(filePath, checksum); + commitEditor.closeDir(); + commitEditor.closeDir(); + SVNCommitInfo info = commitEditor.closeEdit(); + SVNErrorMessage errorMessage = info.getErrorMessage(); - - String filePath = destinationFolder + "/" + fileName; - - ***REMOVED*** if (fileExists(filePath, -1)) { ***REMOVED*** updating existing file - - commitEditor.openFile(filePath, -1); - - ***REMOVED******REMOVED*** else { ***REMOVED*** creating new file - - ***REMOVED***commitEditor.addFile(filePath, null, -1); - ***REMOVED******REMOVED*** - commitEditor.applyTextDelta(filePath, null); - SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); - String checksum = deltaGenerator.sendDelta(filePath, byteArrayInputStream, commitEditor, true); - commitEditor.closeFile(filePath, checksum); - commitEditor.closeDir(); - commitEditor.closeDir(); - commitEditor.closeEdit(); + if (errorMessage != null) + { + this.logger.error("Operation failed: "+errorMessage.getFullMessage()); + SVNCommitException exception = new SVNCommitException(errorMessage); + exception.setFileName(fileName+" to be updated"); + throw exception; ***REMOVED*** + + this.logger.debug("Operation completed"); +***REMOVED*** @@ -197,15 +217,19 @@ public abstract class SVNUpdater { - public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException { - - return new ByteArrayInputStream(FileUtils.readFileToByteArray(file)); -***REMOVED*** +***REMOVED*** public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException +***REMOVED*** { +***REMOVED*** +***REMOVED*** return new ByteArrayInputStream(FileUtils.readFileToByteArray(file)); +***REMOVED******REMOVED*** - private void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) { + private boolean updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) + { + boolean response = false; + try { - System.out.println("Updating algorithm list: " + file); + this.logger.debug("Updating algorithm list: " + file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); @@ -246,15 +270,30 @@ public abstract class SVNUpdater { final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, new ByteArrayInputStream(bytes), commitEditor, true); commitEditor.closeFile(file, checksum); - commitEditor.closeEdit(); + SVNCommitInfo info = commitEditor.closeEdit(); + SVNErrorMessage errorMessage = info.getErrorMessage(); + + if (errorMessage != null) + { + this.logger.error("Operation failed: "+errorMessage.getFullMessage()); + response = false; + ***REMOVED*** + else response = true; + - ***REMOVED*** catch (Exception ex) { - ex.printStackTrace(); + ***REMOVED*** + catch (Exception ex) + { + this.logger.error("Unable to commit algorithm list",ex); + response = false; ***REMOVED*** - finally { + finally + { svnRepository.closeSession(); ***REMOVED*** + + return response; ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java index c5ac255..882545f 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -25,36 +25,51 @@ import org.gcube.common.authorization.library.AuthorizationEntry; ***REMOVED*** import org.gcube.common.resources.gcore.GCoreEndpoint; ***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException; import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.json.JSONArray; import org.json.JSONObject; +***REMOVED*** +***REMOVED*** public class SendMail { +***REMOVED*** + private final String WRITE_MESSAGE_ADDRESS_PATH ="messages/writeMessageToUsers?gcube-token=", + USER_ROLES_ADDRESS_PATH ="2/users/get-usernames-by-role?role-name=DataMiner-Manager&gcube-token=", + ENCODING = "UTF-8", + SENDER_PARAMETER_FORMAT = "&sender=dataminer&recipients=%s&subject=%s&body=%s", + SOCIAL_SERVICE_QUERY_CONDITION ="$resource/Profile/ServiceName/text() eq 'SocialNetworking'", + SOCIAL_SERVICE_URI="jersey-servlet", + JSON_MIME_TYPE = "application/json"; + public SendMail() { - ***REMOVED*** TODO Auto-generated constructor stub + this.logger = LoggerFactory.getLogger(SendMail.class); ***REMOVED*** - public void sendNotification(String subject, String body) throws EMailException { - - ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get()); - - ***REMOVED***String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); - - - + private String getRequestMessage (String addressPath) + { String serviceAddress = this.getSocialService(); + StringBuilder requestMessageBuilder = new StringBuilder(serviceAddress); + if (!serviceAddress.endsWith("/")) requestMessageBuilder.append('/'); + + requestMessageBuilder.append(addressPath).append(SecurityTokenProvider.instance.get()); + String requestForMessage = requestMessageBuilder.toString(); + this.logger.debug("Request "+requestForMessage); + return requestForMessage; +***REMOVED*** + + public void sendNotification(String subject, String body) throws EMailException + { + this.logger.debug("Sending mail notification for "+subject); + this.logger.debug("Body "+body); + - if (!serviceAddress.endsWith("/")) - serviceAddress = serviceAddress + "/"; - String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" - + SecurityTokenProvider.instance.get(); - requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage); @@ -62,17 +77,20 @@ public class SendMail { try { - subject = URLEncoder.encode(subject, "UTF-8"); - body = URLEncoder.encode(body, "UTF-8"); + subject = URLEncoder.encode(subject, ENCODING); + body = URLEncoder.encode(body, ENCODING); ***REMOVED*** catch (UnsupportedEncodingException e) { throw new EMailException(e); ***REMOVED*** - String requestParameters = "&sender=dataminer&recipients=" + this.getAdmins() + "&subject=" + subject + "&body=" - + body; + String requestForMessage = getRequestMessage(WRITE_MESSAGE_ADDRESS_PATH); + requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); + + String requestParameters = String.format(SENDER_PARAMETER_FORMAT, this.getAdmins(), subject , body); + String response = this.sendPostRequest(requestForMessage, requestParameters); ***REMOVED***AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK "); @@ -88,7 +106,7 @@ public class SendMail { public String username(String token) throws ObjectNotFound, Exception { AuthorizationEntry entry = authorizationService().get(token); - System.out.println(entry.getClientInfo().getId()); + this.logger.debug(entry.getClientInfo().getId()); return entry.getClientInfo().getId(); ***REMOVED*** @@ -115,10 +133,10 @@ public class SendMail { public String getSocialService() { SimpleQuery query = queryFor(GCoreEndpoint.class); - query.addCondition("$resource/Profile/ServiceName/text() eq 'SocialNetworking'"); + query.addCondition(SOCIAL_SERVICE_QUERY_CONDITION); DiscoveryClient client = clientFor(GCoreEndpoint.class); List resources = client.submit(query); - String a = resources.get(0).profile().endpointMap().get("jersey-servlet").uri().toString(); + String a = resources.get(0).profile().endpointMap().get(SOCIAL_SERVICE_URI).uri().toString(); return a; ***REMOVED*** @@ -172,21 +190,20 @@ public class SendMail { ***REMOVED***GcoreEndpointReader ep = new GcoreEndpointReader(ScopeProvider.instance.get()); - String serviceAddress = this.getSocialService(); + String serviceAddress = getRequestMessage(USER_ROLES_ADDRESS_PATH); ***REMOVED***String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); ***REMOVED***String serviceAddress = ep.getResourceEntyName();***REMOVED***"https:***REMOVED***socialnetworking1.d4science.org/social-networking-library-ws/rest/"; - serviceAddress = serviceAddress.endsWith("/") ? serviceAddress : serviceAddress + "/"; - serviceAddress+= "2/users/get-usernames-by-role?"; - serviceAddress+= "role-name=DataMiner-Manager" + "&gcube-token=" + SecurityTokenProvider.instance.get(); +***REMOVED*** serviceAddress = serviceAddress.endsWith("/") ? serviceAddress : serviceAddress + "/"; +***REMOVED*** serviceAddress+= "2/users/get-usernames-by-role?role-name=DataMiner-Manager&gcube-token=" + SecurityTokenProvider.instance.get(); - System.out.println("Url is " + serviceAddress); + this.logger.debug("Admin roles url is " + serviceAddress); CloseableHttpClient client = HttpClientBuilder.create().build(); HttpGet getReq = new HttpGet(serviceAddress); - getReq.setHeader("accept", "application/json"); - getReq.setHeader("content-type", "application/json"); + getReq.setHeader("accept", JSON_MIME_TYPE); + getReq.setHeader("content-type", JSON_MIME_TYPE); System.out.println(EntityUtils.toString(client.execute(getReq).getEntity())); return EntityUtils.toString(client.execute(getReq).getEntity()); @@ -210,7 +227,7 @@ public class SendMail { ***REMOVED*** return s.toString().replace("[", "").replace("]", ""); ***REMOVED*** - catch(Exception a){return "ciro.formisano, lucio.lelii, roberto.cirillo, gianpaolo.coro, giancarlo.panichi, scarponi"; ***REMOVED*** + catch(Exception a){return DMPMClientConfiguratorManager.getInstance().getDefaultAdmins(); ***REMOVED*** ***REMOVED*** @@ -240,6 +257,7 @@ public class SendMail { public static void main(String[] args) throws Exception { + ***REMOVED***ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); ***REMOVED***SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java new file mode 100755 index 0000000..a40b2c3 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/AlgorithmException.java @@ -0,0 +1,26 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.exception; + +public class AlgorithmException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -5678597187512954288L; + + public AlgorithmException () + { + super (); +***REMOVED*** + + public AlgorithmException (String message) + { + super (message); +***REMOVED*** + + public AlgorithmException (String message, Throwable cause) + { + super (message, cause); +***REMOVED*** + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java new file mode 100755 index 0000000..7638bce --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/exception/SVNCommitException.java @@ -0,0 +1,41 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util.exception; + +import org.tmatesoft.svn.core.SVNErrorMessage; + +public class SVNCommitException extends Exception { + + /** + * + */ + private static final long serialVersionUID = -5225403308313619585L; + + private SVNErrorMessage svnErrorMessage; + + private String fileName; + + public SVNCommitException(SVNErrorMessage errorMessage) { + super ("Unable to commit"); + this.svnErrorMessage = errorMessage; +***REMOVED*** + + public SVNCommitException(String message,SVNErrorMessage errorMessage) { + super (message); + this.svnErrorMessage = errorMessage; +***REMOVED*** + + public SVNErrorMessage getSvnErrorMessage() { + return svnErrorMessage; +***REMOVED*** + + public void setFileName (String fileName) + { + this.fileName = fileName; +***REMOVED*** + + public String getFileName () + { + return this.fileName; +***REMOVED*** + + +***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java index 9967ccc..fa22a76 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodProduction.java @@ -2,202 +2,28 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import java.io.File; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; -import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -***REMOVED*** -***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException; +import org.tmatesoft.svn.core.SVNException; -import com.jcraft.jsch.Channel; -import com.jcraft.jsch.ChannelSftp; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.Session; +import com.jcraft.jsch.JSchException; import com.jcraft.jsch.SftpException; public class CheckMethodProduction extends CheckMethod{ - -***REMOVED*** + public CheckMethodProduction() { - this.logger = LoggerFactory.getLogger(CheckMethodProduction.class); -***REMOVED*** - - - - - @Override - public boolean algoExists(Algorithm algo/*, String env*/) throws Exception{ - - this.logger.debug("Looking if algo "+algo.getName()+ " exists in production"); - Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); - File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar"); - File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar"); - + super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration()); - System.out.println("First file is located to: "+file.getPath()); - System.out.println("Second file is located to: "+file2.getPath()); - - - if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/))){ - this.copyFromDmToSVN(file/*,env*/); - this.copyFromDmToSVN(file2/*,env*/); - - - return true; - ***REMOVED*** - else - System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); - return false; -***REMOVED*** - - @Override - public void deleteFiles(Algorithm a/*,String env*/) throws Exception - { - JSch jsch = new JSch(); - Session session = null; - Channel channel = null; - ChannelSftp c = null; - Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); - System.out.println("checking existing in env: " + productionConfiguration.getHost()); - - File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar"); - File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); - - - System.out.println("First file is located to: "+file.getPath()); - System.out.println("Second file is located to: "+file2.getPath()); - - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", productionConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - channel = session.openChannel("sftp"); - channel.connect(); - System.out.println("shell channel connected...."); - - c = (ChannelSftp) channel; - - if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){ - - c.rm(file.getPath()); - c.rm(file2.getPath()); - System.out.println("Both the files have been deleted"); - ***REMOVED*** - else System.out.println("Files not found"); - channel.disconnect(); - c.disconnect(); - session.disconnect(); - ***REMOVED*** @Override - public boolean doesExist(String path/*, String env*/) throws Exception { - JSch jsch = new JSch(); - Session session = null; - Channel channel = null; - ChannelSftp c = null; - boolean success = false; - Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); - - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", productionConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - channel = session.openChannel("sftp"); - channel.connect(); - System.out.println("shell channel connected...."); - - c = (ChannelSftp) channel; - - - System.out.println(path); - - try { - c.lstat(path); - success = true; - ***REMOVED*** catch (SftpException e) { - if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { - ***REMOVED*** file doesn't exist - success = false; - ***REMOVED*** - ***REMOVED***success = true; ***REMOVED*** something else went wrong - ***REMOVED*** - channel.disconnect(); - c.disconnect(); - session.disconnect(); - return success; - -***REMOVED*** - - - @Override - public void copyFromDmToSVN(File a/*,String env*/) throws Exception { - JSch jsch = new JSch(); - Session session = null; - SVNUpdater svnUpdater = new SVNUpdaterProduction(); - Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration(); - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", productionConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); + protected void copyFromDmToSVN(File a) throws SVNException, SVNCommitException, JSchException, SftpException { + super.copyFromDmToSVN(a, new SVNUpdaterProduction()); - Channel channel = session.openChannel("sftp"); - channel.connect(); - - ChannelSftp sftp = (ChannelSftp) channel; - - sftp.cd(productionConfiguration.getGhostAlgoDirectory()); - - System.out.println("REMOTE : "+productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()); - System.out.println("LOCAL : /tmp/"+a.getName()); - - sftp.get(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName()); - - channel.disconnect(); - session.disconnect(); - - File f = new File("/tmp/"+a.getName()); - svnUpdater.updateAlgorithmFiles(f); - f.delete(); ***REMOVED*** @@ -242,8 +68,15 @@ public class CheckMethodProduction extends CheckMethod{ ***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", ***REMOVED*** "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462")); -System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org", - "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")); + try + { + a.checkMethod("dataminer-ghost-d.dev.d4science.org", + "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); +***REMOVED*** catch (Exception e) + { + e.printStackTrace(); +***REMOVED*** + ***REMOVED***Algorithm aa = new Algorithm(); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java index 7ceb380..2bebd43 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java @@ -2,218 +2,34 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl; import java.io.File; -import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; -import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; -import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -***REMOVED*** -***REMOVED*** - -import com.jcraft.jsch.Channel; -import com.jcraft.jsch.ChannelSftp; -import com.jcraft.jsch.JSch; -import com.jcraft.jsch.Session; -import com.jcraft.jsch.SftpException; public class CheckMethodStaging extends CheckMethod{ - -***REMOVED*** + public CheckMethodStaging() { - this.logger = LoggerFactory.getLogger(CheckMethodStaging.class); + super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration()); ***REMOVED*** - @Override - public boolean algoExists(Algorithm algo/*, String env*/) throws Exception{ - this.logger.debug("Looking if algo "+algo.getName()+ " exists"); - Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); - File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar"); - File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar"); - this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath()); - - System.out.println("First file is located to: "+file.getPath()); - System.out.println("Second file is located to: "+file2.getPath()); - - - if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/))) - { - this.logger.debug("Files found"); - this.copyFromDmToSVN(file/*,env*/); - this.copyFromDmToSVN(file2/*,env*/); - System.out.println("Files have been copied to SVN"); - - - return true; - ***REMOVED*** - else - { - this.logger.debug("Files not found"); - System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path"); - return false; - ***REMOVED*** - -***REMOVED*** - @Override - public void deleteFiles(Algorithm a/*,String env*/) throws Exception{ - JSch jsch = new JSch(); - Session session = null; - Channel channel = null; - ChannelSftp c = null; - Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); - System.out.println("checking existing in env: " + stagingConfiguration.getHost()); - - File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar"); - File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); - - - System.out.println("First file is located to: "+file.getPath()); - System.out.println("Second file is located to: "+file2.getPath()); - - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - session = jsch.getSession("root", stagingConfiguration.getHost()); - System.out.println("session created."); - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - channel = session.openChannel("sftp"); - channel.connect(); - System.out.println("shell channel connected...."); - - c = (ChannelSftp) channel; - - if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){ - - c.rm(file.getPath()); - c.rm(file2.getPath()); - System.out.println("Both the files have been deleted"); - ***REMOVED*** - else System.out.println("Files not found"); - channel.disconnect(); - c.disconnect(); - session.disconnect(); - -***REMOVED*** - - - @Override - public boolean doesExist(String path/*, String env*/) throws Exception { - this.logger.debug("Looking if file "+path + " exists"); - JSch jsch = new JSch(); - Session session = null; - Channel channel = null; - ChannelSftp c = null; - boolean success = false; - Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - this.logger.debug("Staging configuration host "+stagingConfiguration.getHost()); - session = jsch.getSession("root",stagingConfiguration.getHost() ); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); - - channel = session.openChannel("sftp"); - channel.connect(); - System.out.println("shell channel connected...."); - - c = (ChannelSftp) channel; - - ***REMOVED***SftpATTRS is = null; - System.out.println(path); - - try { - c.lstat(path); - this.logger.debug("File found"); - success = true; - ***REMOVED*** catch (SftpException e) - { - this.logger.error("File not found",e); - - if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) - { - ***REMOVED*** file doesn't exist - success = false; - ***REMOVED*** - ***REMOVED***success = true; ***REMOVED*** something else went wrong - ***REMOVED*** - channel.disconnect(); - c.disconnect(); - session.disconnect(); - this.logger.debug("Operation result "+success); - return success; - -***REMOVED*** @Override - public void copyFromDmToSVN(File a/*,String env*/) throws Exception { - JSch jsch = new JSch(); - Session session = null; - SVNUpdater svnUpdater = new SVNUpdaterStaging(); - Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration(); - - - jsch.setKnownHosts("~/.ssh/known_hosts"); - String privateKey = "~/.ssh/id_rsa"; - - jsch.addIdentity(privateKey); - System.out.println("Private Key Added."); - - session = jsch.getSession("root", stagingConfiguration.getHost()); - System.out.println("session created."); - - java.util.Properties config = new java.util.Properties(); - config.put("StrictHostKeyChecking", "no"); - - session.setConfig(config); - session.connect(); + protected void copyFromDmToSVN(File a) throws Exception{ + super.copyFromDmToSVN(a, new SVNUpdaterStaging()); - Channel channel = session.openChannel("sftp"); - channel.connect(); - - ChannelSftp sftp = (ChannelSftp) channel; - - sftp.cd(stagingConfiguration.getGhostAlgoDirectory()); - - System.out.println("REMOTE : "+stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()); - System.out.println("LOCAL : /tmp/"+a.getName()); - - sftp.get(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName()); - - channel.disconnect(); - session.disconnect(); - - File f = new File("/tmp/"+a.getName()); - svnUpdater.updateAlgorithmFiles(f); - f.delete(); ***REMOVED*** - + @@ -258,8 +74,15 @@ public class CheckMethodStaging extends CheckMethod{ ***REMOVED***System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", ***REMOVED*** "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462")); -System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org", - "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")); + try + { + a.checkMethod("dataminer-ghost-d.dev.d4science.org", + "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); +***REMOVED*** catch (Exception e) + { + e.printStackTrace(); +***REMOVED*** + ***REMOVED***Algorithm aa = new Algorithm(); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java index b29311e..5f103c0 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperProduction.java @@ -8,22 +8,11 @@ public class NotificationHelperProduction extends NotificationHelper{ ***REMOVED*** private Exception executionException; + public NotificationHelperProduction() { + super ("[DataMinerGhostProductionInstallationRequestReport]"); +***REMOVED*** - private String getSubjectHeader() { - return "[DataMinerGhostProductionInstallationRequestReport]"; - ***REMOVED*** - @Override - public String getSuccessSubject() { - return this.getSubjectHeader()+" is SUCCESS"; - ***REMOVED*** - - - - @Override - public String getFailedSubject() { - return String.format(this.getSubjectHeader()+" is FAILED"); - ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java index eace46a..cacf7d4 100755 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/NotificationHelperStaging.java @@ -8,25 +8,11 @@ public class NotificationHelperStaging extends NotificationHelper { ***REMOVED*** private Exception executionException; + public NotificationHelperStaging() { + super ("[DataMinerGhostStagingInstallationRequestReport]"); + ***REMOVED*** + - private String getSubjectHeader() { - return "[DataMinerGhostStagingInstallationRequestReport]"; - ***REMOVED*** - - - @Override - public String getSuccessSubject() { - return this.getSubjectHeader()+" is SUCCESS"; - ***REMOVED*** - - - - - @Override - public String getFailedSubject() { - return String.format(this.getSubjectHeader()+" is FAILED"); - ***REMOVED*** - ***REMOVED*** diff --git a/src/main/resources/default.admins b/src/main/resources/default.admins new file mode 100755 index 0000000..4d3d2b5 --- /dev/null +++ b/src/main/resources/default.admins @@ -0,0 +1,6 @@ +1=ciro.formisano +2=lucio.lelii +3=roberto.cirillo +4=gianpaolo.coro +5=giancarlo.panichi +6=scarponi \ No newline at end of file