This commit is contained in:
Ciro Formisano 2017-12-15 12:01:42 +00:00
parent b51f57304f
commit 0ec3e13515
14 changed files with 513 additions and 538 deletions

View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataAnalysis</groupId> <groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId> <artifactId>dataminer-pool-manager</artifactId>
<packaging>war</packaging> <packaging>war</packaging>
<version>2.3.0-SNAPSHOT</version> <version>2.4.0-SNAPSHOT</version>
<name>dataminer-pool-manager</name> <name>dataminer-pool-manager</name>
<description> <description>

View File

@ -1,6 +1,8 @@
package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration; package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration;
import java.io.IOException; import java.io.IOException;
import java.util.Iterator;
import java.util.Properties;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.ScopedCacheMap; import org.gcube.dataanalysis.dataminer.poolmanager.clients.ScopedCacheMap;
@ -11,6 +13,8 @@ import org.tmatesoft.svn.core.SVNException;
public class DMPMClientConfiguratorManager{ public class DMPMClientConfiguratorManager{
private final Logger logger; private final Logger logger;
private Properties defaultAdmins;
private String admins;
static DMPMClientConfiguratorManager instance; static DMPMClientConfiguratorManager instance;
@ -18,8 +22,19 @@ public class DMPMClientConfiguratorManager{
private DMPMClientConfiguratorManager () private DMPMClientConfiguratorManager ()
{ {
this.admins = null;
this.cacheMap = new ScopedCacheMap(); this.cacheMap = new ScopedCacheMap();
this.logger = LoggerFactory.getLogger(DMPMClientConfiguratorManager.class); this.logger = LoggerFactory.getLogger(DMPMClientConfiguratorManager.class);
this.defaultAdmins = new Properties();
try
{
this.defaultAdmins.load(this.getClass().getResourceAsStream("/default.admins"));
this.logger.debug("Default users successfully loaded");
} catch (Exception e)
{
this.logger.error("Unable to get default users",e);
}
} }
private ClientConfigurationCache getCurrentCache () private ClientConfigurationCache getCurrentCache ()
@ -60,6 +75,29 @@ public class DMPMClientConfiguratorManager{
return new ConfigurationImpl(CONFIGURATIONS.STAGE, getCurrentCache()); return new ConfigurationImpl(CONFIGURATIONS.STAGE, getCurrentCache());
} }
public String getDefaultAdmins ()
{
if (this.admins == null && this.defaultAdmins.isEmpty()) this.admins= "ciro.formisano";
else if (admins == null)
{
Iterator<Object> keys = this.defaultAdmins.keySet().iterator();
StringBuilder response = new StringBuilder();
while (keys.hasNext())
{
String key = (String) keys.next();
response.append(this.defaultAdmins.getProperty(key)).append(", ");
}
this.admins = response.substring(0, response.length()-2);
}
this.logger.debug("Default admins list "+this.admins);
return this.admins;
}
public static void main(String[] args) throws IOException, SVNException { public static void main(String[] args) throws IOException, SVNException {
DMPMClientConfiguratorManager a = new DMPMClientConfiguratorManager(); DMPMClientConfiguratorManager a = new DMPMClientConfiguratorManager();
ScopeProvider.instance.set("/gcube/devNext/NextNext"); ScopeProvider.instance.set("/gcube/devNext/NextNext");

View File

@ -19,7 +19,9 @@ import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.AlgorithmException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
@ -186,18 +188,34 @@ public abstract class DMPMJob {
//this.setStatusInformation(STATUS.PROGRESS); //this.setStatusInformation(STATUS.PROGRESS);
this.logger.debug("Checking the method..."); this.logger.debug("Checking the method...");
if (methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get())&&(methodChecker.algoExists(this.algorithm))) try
{ {
methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get());
methodChecker.copyAlgorithms(this.algorithm);
this.logger.debug("Method OK and algo exists"); this.logger.debug("Method OK and algo exists");
System.out.println("Interface check ok!"); this.logger.debug("Interface check ok!");
System.out.println("Both the files exist at the correct path!"); this.logger.debug("Both the files exist at the correct path!");
boolean algorithmListResult = this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.vREName,this.category, this.algorithm_type,
this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.vREName,this.category, this.algorithm_type,
this.algorithm.getFullname()); this.algorithm.getFullname());
this.setStatusInformation(STATUS.COMPLETED); this.setStatusInformation(STATUS.COMPLETED);
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
return; String bodyResponse = algorithmListResult ? nh.getSuccessBody("\n\n"+this.buildInfo()) : nh.getSuccessBody("\n\n"+this.buildInfo())+"\nWARNING: algorithm list could not be updated on SVN";
} else
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", bodyResponse);
} catch (SVNCommitException e)
{
this.logger.error("Unable to complete the commit operation: "+e.getMessage());
this.setStatusInformation(STATUS.FAILED);
sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",
nh.getFailedBody(
"\n"+
"Commit operation failed for "+e.getFileName()
+ "the message of the SVN Server is the following:\n"+e.getSvnErrorMessage().getMessage()+"\n\n"
+this.buildInfo()));
} catch (AlgorithmException ae)
{ {
this.logger.debug("Operation failed"); this.logger.debug("Operation failed");
this.setStatusInformation(STATUS.FAILED); this.setStatusInformation(STATUS.FAILED);
@ -207,8 +225,8 @@ public abstract class DMPMJob {
"Installation completed but DataMiner Interface not working correctly or files " "Installation completed but DataMiner Interface not working correctly or files "
+ this.algorithm.getName() + ".jar and " + this.algorithm.getName() + this.algorithm.getName() + ".jar and " + this.algorithm.getName()
+ "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo()); + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
} }
} }
} }

View File

@ -9,28 +9,50 @@ import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Properties;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.AlgorithmException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNException;
import com.jcraft.jsch.Channel;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
public abstract class CheckMethod { public abstract class CheckMethod {
private Logger logger; private Logger logger;
private Configuration configuration;
public CheckMethod() private final String KNOWN_HOSTS= "~/.ssh/known_hosts",
PRIVATE_KEY = "~/.ssh/id_rsa",
SSH_USER = "root",
SFTP_PROTOCOL = "sftp",
TEMP_DIRECTORY = "tmp";
private final Properties sshConfig;
public CheckMethod(Configuration configuration)
{ {
this.logger = LoggerFactory.getLogger(CheckMethod.class); this.logger = LoggerFactory.getLogger(CheckMethod.class);
sshConfig = new java.util.Properties();
sshConfig.put("StrictHostKeyChecking", "no");
} }
public boolean checkMethod(String machine, String token) throws Exception { public void checkMethod(String machine, String token) throws AlgorithmException {
try { try {
this.logger.debug("Checking method for machine "+machine); this.logger.debug("Checking method for machine "+machine);
this.logger.debug("By using tocken "+token); this.logger.debug("By using tocken "+token);
System.out.println("Machine: " + machine); this.logger.debug("Machine: " + machine);
// String getCapabilitesRequest = new String(); // String getCapabilitesRequest = new String();
// String getCapabilitesResponse = new String(); // String getCapabilitesResponse = new String();
System.out.println(" Token: " + token); this.logger.debug(" Token: " + token);
String request = "http://" + machine String request = "http://" + machine
+ "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token; + "/wps/WebProcessingService?Request=GetCapabilities&Service=WPS&gcube-token=" + token;
String response = machine + "___" + token + ".xml"; String response = machine + "___" + token + ".xml";
@ -72,12 +94,12 @@ public abstract class CheckMethod {
String operatorDescription = innerLine.substring(innerLine.indexOf(">") + 1); String operatorDescription = innerLine.substring(innerLine.indexOf(">") + 1);
operatorDescription = operatorDescription.substring(0, operatorDescription.indexOf("<")); operatorDescription = operatorDescription.substring(0, operatorDescription.indexOf("<"));
this.logger.debug("Operator descriptor "+operatorDescription); this.logger.debug("Operator descriptor "+operatorDescription);
System.out.println(" " + operatorDescription); this.logger.debug(" " + operatorDescription);
innerFlag = false; innerFlag = false;
} else if (innerLine.contains("ows:ExceptionText")) } else if (innerLine.contains("ows:ExceptionText"))
{ {
this.logger.debug("Exception found"); this.logger.debug("Exception found");
System.out.println(" " + "error retrieving operator description"); this.logger.debug(" " + "error retrieving operator description");
innerFlag = false; innerFlag = false;
flag = false; flag = false;
} else } else
@ -91,29 +113,152 @@ public abstract class CheckMethod {
this.logger.debug("Operation successful"); this.logger.debug("Operation successful");
fileWriter.close(); fileWriter.close();
return true;
} catch (Exception e) { } catch (Exception e) {
e.getMessage();
this.logger.error("Error "+e.getMessage(),e); throw new AlgorithmException("Error "+e.getMessage(),e);
return false;
} }
} }
public abstract boolean algoExists(Algorithm algo/*, String env*/) throws Exception; public void copyAlgorithms(Algorithm algo/*, String env*/) throws SVNCommitException, Exception{
this.logger.debug("Looking if algo "+algo.getName()+ " exists");
File file = new File(this.configuration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
File file2 = new File(this.configuration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath());
public abstract void deleteFiles(Algorithm a/*,String env*/) throws Exception; if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))
{
this.logger.debug("Files found");
this.copyFromDmToSVN(file/*,env*/);
this.copyFromDmToSVN(file2/*,env*/);
this.logger.debug("Files have been copied to SVN");
}
else
{
this.logger.debug("Files not found");
this.logger.debug("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
throw new AlgorithmException("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
}
}
public abstract boolean doesExist(String path/*, String env*/) throws Exception;
public abstract void copyFromDmToSVN(File a/*,String env*/) throws Exception; public void deleteFiles(Algorithm a/*,String env*/) throws Exception
{
Session session = generateSession();
System.out.println("checking existing in env: " + this.configuration.getHost());
File file = new File(this.configuration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(this.configuration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
System.out.println("session created.");
session.setConfig(this.sshConfig);
session.connect();
Channel channel = session.openChannel(SFTP_PROTOCOL);
channel.connect();
System.out.println("shell channel connected....");
ChannelSftp c = (ChannelSftp) channel;
if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
}
else System.out.println("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
}
public boolean doesExist(String path/*, String env*/) throws Exception {
Session session = generateSession();
boolean success = false;
session.connect();
Channel channel = session.openChannel(SFTP_PROTOCOL);
channel.connect();
System.out.println("shell channel connected....");
ChannelSftp c = (ChannelSftp) channel;
System.out.println(path);
try {
c.lstat(path);
success = true;
} catch (SftpException e) {
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
// file doesn't exist
success = false;
}
//success = true; // something else went wrong
}
channel.disconnect();
c.disconnect();
session.disconnect();
this.logger.debug("Operation result "+success);
return success;
}
protected abstract void copyFromDmToSVN(File a) throws SVNCommitException, Exception;
protected void copyFromDmToSVN(File algorithmsFile/*,String env*/,SVNUpdater svnUpdater) throws SVNException, SVNCommitException, JSchException, SftpException {
this.logger.debug("Copying algorithm file from Data Miner to SVN");
String fileName = algorithmsFile.getName();
this.logger.debug("File name "+fileName);
Session session = generateSession();
session.connect();
Channel channel = session.openChannel(SFTP_PROTOCOL);
channel.connect();
ChannelSftp sftp = (ChannelSftp) channel;
sftp.cd(this.configuration.getGhostAlgoDirectory());
String remoteFile = new StringBuilder(this.configuration.getGhostAlgoDirectory()).append(File.separator).append(fileName).toString();
this.logger.debug("Remote file "+remoteFile);
String localFile = new StringBuilder(File.separator).append(TEMP_DIRECTORY).append(File.separator).append(fileName).toString();
this.logger.debug("Local file "+localFile);
sftp.get(remoteFile,localFile);
channel.disconnect();
session.disconnect();
File f = new File(localFile);
svnUpdater.updateAlgorithmFiles(f);
f.delete();
}
private Session generateSession () throws JSchException
{
JSch jsch = new JSch();
jsch.setKnownHosts(KNOWN_HOSTS);
jsch.addIdentity(PRIVATE_KEY);
this.logger.debug("Private Key Added.");
Session session = jsch.getSession(SSH_USER, this.configuration.getHost());
this.logger.debug("session created.");
session.setConfig(this.sshConfig);
return session;
}

View File

@ -4,6 +4,13 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util;
public abstract class NotificationHelper { public abstract class NotificationHelper {
private String subjectHeader;
protected NotificationHelper (String subjectHeader)
{
this.subjectHeader = subjectHeader;
}
// private Exception executionException; // private Exception executionException;
@ -17,10 +24,16 @@ public abstract class NotificationHelper {
// this.executionException = executionException; // this.executionException = executionException;
// } // }
public abstract String getSuccessSubject(); public String getSuccessSubject() {
return this.subjectHeader+" is SUCCESS";
}
public abstract String getFailedSubject();
public String getFailedSubject() {
return String.format(this.subjectHeader+" is FAILED");
}
public String getSuccessBody(String info) { public String getSuccessBody(String info) {

View File

@ -3,6 +3,8 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException; import java.io.IOException;
import java.text.DateFormat; import java.text.DateFormat;
import java.text.ParseException; import java.text.ParseException;
@ -19,10 +21,14 @@ import java.util.List;
import java.util.Set; import java.util.Set;
import java.util.TimeZone; import java.util.TimeZone;
import org.apache.commons.io.FileUtils;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNCommitInfo;
import org.tmatesoft.svn.core.SVNErrorMessage;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNNodeKind; import org.tmatesoft.svn.core.SVNNodeKind;
import org.tmatesoft.svn.core.internal.wc.SVNFileUtil; import org.tmatesoft.svn.core.internal.wc.SVNFileUtil;
@ -38,11 +44,12 @@ public abstract class SVNUpdater {
private SVNRepository svnRepository; private SVNRepository svnRepository;
private Configuration configuration; private Configuration configuration;
private Logger logger;
public SVNUpdater(Configuration configuration) throws SVNException { public SVNUpdater(Configuration configuration) throws SVNException {
this.configuration = configuration; this.configuration = configuration;
this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository(); this.svnRepository = SVNRepositoryManager.getInstance(configuration).getSvnRepository();
this.logger = LoggerFactory.getLogger(SVNUpdater.class);
} }
// public void updateRPRotoDeps(Algorithm algorithm) { // public void updateRPRotoDeps(Algorithm algorithm) {
@ -64,7 +71,10 @@ public abstract class SVNUpdater {
private String getDependencyFile (Configuration configuration, String language) private String getDependencyFile (Configuration configuration, String language)
{ {
switch (language) { this.logger.debug("Getting dependency file for language "+language);
switch (language)
{
case "R": case "R":
return configuration.getSVNCRANDepsList(); return configuration.getSVNCRANDepsList();
case "R-blackbox": case "R-blackbox":
@ -89,14 +99,14 @@ public abstract class SVNUpdater {
} }
public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) public boolean updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/)
{ {
this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); return this.updateSVNAlgorithmList(this.configuration.getSVNAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
} }
public void updateAlgorithmFiles(File a) throws SVNException{ public void updateAlgorithmFiles(File a) throws SVNException, SVNCommitException{
//this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); //this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a);
this.updateAlgorithmList(this.configuration.getRepository(), a); this.updateAlgorithmList(this.configuration.getRepository(), a);
} }
@ -106,83 +116,93 @@ public abstract class SVNUpdater {
private void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException { private void updateAlgorithmList(String svnMainAlgoRepo, File algorithmsFile) throws SVNException, SVNCommitException
try { {
System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo); this.logger.debug("Adding .jar file: " + algorithmsFile + " to repository " + svnMainAlgoRepo);
if (fileExists(svnMainAlgoRepo+File.separator+a.getName(), -1)){ try
this.updateFile(reteriveByteArrayInputStream(a), svnMainAlgoRepo, a.getName()); {
if (fileExists(svnMainAlgoRepo+File.separator+algorithmsFile.getName(), -1))
{
this.updateFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo, algorithmsFile.getName());
} }
else this.putFile(reteriveByteArrayInputStream(a), svnMainAlgoRepo,a.getName()); else this.putFile(new FileInputStream(algorithmsFile), svnMainAlgoRepo,algorithmsFile.getName());
} catch (Exception ex) {
ex.printStackTrace();
} }
catch (FileNotFoundException e)
finally { {
svnRepository.closeSession(); this.logger.error("Temporary algorithm file not found: this exception should not happen",e);
} }
} finally
{
this.svnRepository.closeSession();
}
}
public void putFile(ByteArrayInputStream byteArrayInputStream, String destinationFolder, String fileName) public void putFile(FileInputStream fileInputSreeam, String destinationFolder, String fileName) throws SVNException, SVNCommitException
throws SVNException { {
this.logger.debug("Putting new file on the SVN repository");
final ISVNEditor commitEditor = svnRepository.getCommitEditor("Add algorithm to list", null); final ISVNEditor commitEditor = svnRepository.getCommitEditor("Add algorithm to list", null);
commitEditor.openRoot(-1); commitEditor.openRoot(-1);
commitEditor.openDir(destinationFolder, -1); commitEditor.openDir(destinationFolder, -1);
String filePath = destinationFolder + "/" + fileName;
commitEditor.addFile(filePath, null, -1);
commitEditor.applyTextDelta(filePath, null);
SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
String checksum = deltaGenerator.sendDelta(filePath, fileInputSreeam, commitEditor, true);
commitEditor.closeFile(filePath, checksum);
commitEditor.closeDir();
commitEditor.closeDir();
SVNCommitInfo info = commitEditor.closeEdit();
SVNErrorMessage errorMessage = info.getErrorMessage();
if (errorMessage != null)
String filePath = destinationFolder + "/" + fileName; {
this.logger.error("Operation failed: "+errorMessage.getFullMessage());
SVNCommitException exception = new SVNCommitException(errorMessage);
//commitEditor.openFile(filePath, -1); exception.setFileName(fileName);
throw exception;
commitEditor.addFile(filePath, null, -1);
commitEditor.applyTextDelta(filePath, null);
SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
String checksum = deltaGenerator.sendDelta(filePath, byteArrayInputStream, commitEditor, true);
commitEditor.closeFile(filePath, checksum);
commitEditor.closeDir();
commitEditor.closeDir();
commitEditor.closeEdit();
} }
this.logger.debug("Operation completed");
public void updateFile(ByteArrayInputStream byteArrayInputStream, String destinationFolder, String fileName) }
throws SVNException {
final ISVNEditor commitEditor = svnRepository.getCommitEditor("Add algorithm to list", null); public void updateFile(FileInputStream fileInputStream, String destinationFolder, String fileName) throws SVNException, SVNCommitException {
this.logger.debug("Updating existing file on the SVN repository");
final ISVNEditor commitEditor = svnRepository.getCommitEditor("Updating algorithm", null);
commitEditor.openRoot(-1); commitEditor.openRoot(-1);
commitEditor.openDir(destinationFolder, -1); commitEditor.openDir(destinationFolder, -1);
String filePath = destinationFolder + "/" + fileName;
// if (fileExists(filePath, -1)) { // updating existing file
commitEditor.openFile(filePath, -1);
//} else { // creating new file
//commitEditor.addFile(filePath, null, -1);
//}
commitEditor.applyTextDelta(filePath, null);
SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
String checksum = deltaGenerator.sendDelta(filePath, fileInputStream, commitEditor, true);
commitEditor.closeFile(filePath, checksum);
commitEditor.closeDir();
commitEditor.closeDir();
SVNCommitInfo info = commitEditor.closeEdit();
SVNErrorMessage errorMessage = info.getErrorMessage();
if (errorMessage != null)
String filePath = destinationFolder + "/" + fileName; {
this.logger.error("Operation failed: "+errorMessage.getFullMessage());
// if (fileExists(filePath, -1)) { // updating existing file SVNCommitException exception = new SVNCommitException(errorMessage);
exception.setFileName(fileName+" to be updated");
commitEditor.openFile(filePath, -1); throw exception;
//} else { // creating new file
//commitEditor.addFile(filePath, null, -1);
//}
commitEditor.applyTextDelta(filePath, null);
SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
String checksum = deltaGenerator.sendDelta(filePath, byteArrayInputStream, commitEditor, true);
commitEditor.closeFile(filePath, checksum);
commitEditor.closeDir();
commitEditor.closeDir();
commitEditor.closeEdit();
} }
this.logger.debug("Operation completed");
}
@ -197,15 +217,19 @@ public abstract class SVNUpdater {
public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException { // public static ByteArrayInputStream reteriveByteArrayInputStream(File file) throws IOException
// {
return new ByteArrayInputStream(FileUtils.readFileToByteArray(file)); //
} // return new ByteArrayInputStream(FileUtils.readFileToByteArray(file));
// }
private void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) { private boolean updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/)
{
boolean response = false;
try { try {
System.out.println("Updating algorithm list: " + file); this.logger.debug("Updating algorithm list: " + file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
@ -246,15 +270,30 @@ public abstract class SVNUpdater {
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true); new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum); commitEditor.closeFile(file, checksum);
commitEditor.closeEdit(); SVNCommitInfo info = commitEditor.closeEdit();
SVNErrorMessage errorMessage = info.getErrorMessage();
} catch (Exception ex) { if (errorMessage != null)
ex.printStackTrace(); {
this.logger.error("Operation failed: "+errorMessage.getFullMessage());
response = false;
}
else response = true;
}
catch (Exception ex)
{
this.logger.error("Unable to commit algorithm list",ex);
response = false;
} }
finally { finally
{
svnRepository.closeSession(); svnRepository.closeSession();
} }
return response;
} }

View File

@ -25,36 +25,51 @@ import org.gcube.common.authorization.library.AuthorizationEntry;
import org.gcube.common.authorization.library.provider.SecurityTokenProvider; import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
import org.gcube.common.resources.gcore.GCoreEndpoint; import org.gcube.common.resources.gcore.GCoreEndpoint;
import org.gcube.common.scope.api.ScopeProvider; import org.gcube.common.scope.api.ScopeProvider;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException;
import org.gcube.resources.discovery.client.api.DiscoveryClient; import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery; import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.json.JSONArray; import org.json.JSONArray;
import org.json.JSONObject; import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class SendMail { public class SendMail {
private Logger logger;
private final String WRITE_MESSAGE_ADDRESS_PATH ="messages/writeMessageToUsers?gcube-token=",
USER_ROLES_ADDRESS_PATH ="2/users/get-usernames-by-role?role-name=DataMiner-Manager&gcube-token=",
ENCODING = "UTF-8",
SENDER_PARAMETER_FORMAT = "&sender=dataminer&recipients=%s&subject=%s&body=%s",
SOCIAL_SERVICE_QUERY_CONDITION ="$resource/Profile/ServiceName/text() eq 'SocialNetworking'",
SOCIAL_SERVICE_URI="jersey-servlet",
JSON_MIME_TYPE = "application/json";
public SendMail() { public SendMail() {
// TODO Auto-generated constructor stub this.logger = LoggerFactory.getLogger(SendMail.class);
} }
public void sendNotification(String subject, String body) throws EMailException { private String getRequestMessage (String addressPath)
{
//AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get());
//String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get());
String serviceAddress = this.getSocialService(); String serviceAddress = this.getSocialService();
StringBuilder requestMessageBuilder = new StringBuilder(serviceAddress);
if (!serviceAddress.endsWith("/")) requestMessageBuilder.append('/');
requestMessageBuilder.append(addressPath).append(SecurityTokenProvider.instance.get());
String requestForMessage = requestMessageBuilder.toString();
this.logger.debug("Request "+requestForMessage);
return requestForMessage;
}
public void sendNotification(String subject, String body) throws EMailException
{
this.logger.debug("Sending mail notification for "+subject);
this.logger.debug("Body "+body);
if (!serviceAddress.endsWith("/"))
serviceAddress = serviceAddress + "/";
String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token="
+ SecurityTokenProvider.instance.get();
requestForMessage = requestForMessage.replace("http://", "https://").replace(":80", "");
//AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage); //AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage);
@ -62,16 +77,19 @@ public class SendMail {
try try
{ {
subject = URLEncoder.encode(subject, "UTF-8"); subject = URLEncoder.encode(subject, ENCODING);
body = URLEncoder.encode(body, "UTF-8"); body = URLEncoder.encode(body, ENCODING);
} }
catch (UnsupportedEncodingException e) catch (UnsupportedEncodingException e)
{ {
throw new EMailException(e); throw new EMailException(e);
} }
String requestParameters = "&sender=dataminer&recipients=" + this.getAdmins() + "&subject=" + subject + "&body=" String requestForMessage = getRequestMessage(WRITE_MESSAGE_ADDRESS_PATH);
+ body; requestForMessage = requestForMessage.replace("http://", "https://").replace(":80", "");
String requestParameters = String.format(SENDER_PARAMETER_FORMAT, this.getAdmins(), subject , body);
String response = this.sendPostRequest(requestForMessage, requestParameters); String response = this.sendPostRequest(requestForMessage, requestParameters);
//AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK "); //AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK ");
@ -88,7 +106,7 @@ public class SendMail {
public String username(String token) throws ObjectNotFound, Exception { public String username(String token) throws ObjectNotFound, Exception {
AuthorizationEntry entry = authorizationService().get(token); AuthorizationEntry entry = authorizationService().get(token);
System.out.println(entry.getClientInfo().getId()); this.logger.debug(entry.getClientInfo().getId());
return entry.getClientInfo().getId(); return entry.getClientInfo().getId();
} }
@ -115,10 +133,10 @@ public class SendMail {
public String getSocialService() { public String getSocialService() {
SimpleQuery query = queryFor(GCoreEndpoint.class); SimpleQuery query = queryFor(GCoreEndpoint.class);
query.addCondition("$resource/Profile/ServiceName/text() eq 'SocialNetworking'"); query.addCondition(SOCIAL_SERVICE_QUERY_CONDITION);
DiscoveryClient<GCoreEndpoint> client = clientFor(GCoreEndpoint.class); DiscoveryClient<GCoreEndpoint> client = clientFor(GCoreEndpoint.class);
List<GCoreEndpoint> resources = client.submit(query); List<GCoreEndpoint> resources = client.submit(query);
String a = resources.get(0).profile().endpointMap().get("jersey-servlet").uri().toString(); String a = resources.get(0).profile().endpointMap().get(SOCIAL_SERVICE_URI).uri().toString();
return a; return a;
} }
@ -172,21 +190,20 @@ public class SendMail {
//GcoreEndpointReader ep = new GcoreEndpointReader(ScopeProvider.instance.get()); //GcoreEndpointReader ep = new GcoreEndpointReader(ScopeProvider.instance.get());
String serviceAddress = this.getSocialService(); String serviceAddress = getRequestMessage(USER_ROLES_ADDRESS_PATH);
//String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); //String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get());
//String serviceAddress = ep.getResourceEntyName();//"https://socialnetworking1.d4science.org/social-networking-library-ws/rest/"; //String serviceAddress = ep.getResourceEntyName();//"https://socialnetworking1.d4science.org/social-networking-library-ws/rest/";
serviceAddress = serviceAddress.endsWith("/") ? serviceAddress : serviceAddress + "/"; // serviceAddress = serviceAddress.endsWith("/") ? serviceAddress : serviceAddress + "/";
serviceAddress+= "2/users/get-usernames-by-role?"; // serviceAddress+= "2/users/get-usernames-by-role?role-name=DataMiner-Manager&gcube-token=" + SecurityTokenProvider.instance.get();
serviceAddress+= "role-name=DataMiner-Manager" + "&gcube-token=" + SecurityTokenProvider.instance.get();
System.out.println("Url is " + serviceAddress); this.logger.debug("Admin roles url is " + serviceAddress);
CloseableHttpClient client = HttpClientBuilder.create().build(); CloseableHttpClient client = HttpClientBuilder.create().build();
HttpGet getReq = new HttpGet(serviceAddress); HttpGet getReq = new HttpGet(serviceAddress);
getReq.setHeader("accept", "application/json"); getReq.setHeader("accept", JSON_MIME_TYPE);
getReq.setHeader("content-type", "application/json"); getReq.setHeader("content-type", JSON_MIME_TYPE);
System.out.println(EntityUtils.toString(client.execute(getReq).getEntity())); System.out.println(EntityUtils.toString(client.execute(getReq).getEntity()));
return EntityUtils.toString(client.execute(getReq).getEntity()); return EntityUtils.toString(client.execute(getReq).getEntity());
@ -210,7 +227,7 @@ public class SendMail {
} }
return s.toString().replace("[", "").replace("]", ""); return s.toString().replace("[", "").replace("]", "");
} }
catch(Exception a){return "ciro.formisano, lucio.lelii, roberto.cirillo, gianpaolo.coro, giancarlo.panichi, scarponi"; } catch(Exception a){return DMPMClientConfiguratorManager.getInstance().getDefaultAdmins(); }
} }
@ -240,6 +257,7 @@ public class SendMail {
public static void main(String[] args) throws Exception { public static void main(String[] args) throws Exception {
//ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); //ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
//SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); //SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");

View File

@ -0,0 +1,26 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
public class AlgorithmException extends Exception {
/**
*
*/
private static final long serialVersionUID = -5678597187512954288L;
public AlgorithmException ()
{
super ();
}
public AlgorithmException (String message)
{
super (message);
}
public AlgorithmException (String message, Throwable cause)
{
super (message, cause);
}
}

View File

@ -0,0 +1,41 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
import org.tmatesoft.svn.core.SVNErrorMessage;
public class SVNCommitException extends Exception {
/**
*
*/
private static final long serialVersionUID = -5225403308313619585L;
private SVNErrorMessage svnErrorMessage;
private String fileName;
public SVNCommitException(SVNErrorMessage errorMessage) {
super ("Unable to commit");
this.svnErrorMessage = errorMessage;
}
public SVNCommitException(String message,SVNErrorMessage errorMessage) {
super (message);
this.svnErrorMessage = errorMessage;
}
public SVNErrorMessage getSvnErrorMessage() {
return svnErrorMessage;
}
public void setFileName (String fileName)
{
this.fileName = fileName;
}
public String getFileName ()
{
return this.fileName;
}
}

View File

@ -2,202 +2,28 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import java.io.File; import java.io.File;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException;
import org.slf4j.Logger; import org.tmatesoft.svn.core.SVNException;
import org.slf4j.LoggerFactory;
import com.jcraft.jsch.Channel; import com.jcraft.jsch.JSchException;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException; import com.jcraft.jsch.SftpException;
public class CheckMethodProduction extends CheckMethod{ public class CheckMethodProduction extends CheckMethod{
private Logger logger;
public CheckMethodProduction() public CheckMethodProduction()
{ {
this.logger = LoggerFactory.getLogger(CheckMethodProduction.class); super (DMPMClientConfiguratorManager.getInstance().getProductionConfiguration());
}
@Override
public boolean algoExists(Algorithm algo/*, String env*/) throws Exception{
this.logger.debug("Looking if algo "+algo.getName()+ " exists in production");
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/))){
this.copyFromDmToSVN(file/*,env*/);
this.copyFromDmToSVN(file2/*,env*/);
return true;
}
else
System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
return false;
}
@Override
public void deleteFiles(Algorithm a/*,String env*/) throws Exception
{
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
System.out.println("checking existing in env: " + productionConfiguration.getHost());
File file = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
}
else System.out.println("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
} }
@Override @Override
public boolean doesExist(String path/*, String env*/) throws Exception { protected void copyFromDmToSVN(File a) throws SVNException, SVNCommitException, JSchException, SftpException {
JSch jsch = new JSch(); super.copyFromDmToSVN(a, new SVNUpdaterProduction());
Session session = null;
Channel channel = null;
ChannelSftp c = null;
boolean success = false;
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
System.out.println(path);
try {
c.lstat(path);
success = true;
} catch (SftpException e) {
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
// file doesn't exist
success = false;
}
//success = true; // something else went wrong
}
channel.disconnect();
c.disconnect();
session.disconnect();
return success;
}
@Override
public void copyFromDmToSVN(File a/*,String env*/) throws Exception {
JSch jsch = new JSch();
Session session = null;
SVNUpdater svnUpdater = new SVNUpdaterProduction();
Configuration productionConfiguration = DMPMClientConfiguratorManager.getInstance().getProductionConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", productionConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
ChannelSftp sftp = (ChannelSftp) channel;
sftp.cd(productionConfiguration.getGhostAlgoDirectory());
System.out.println("REMOTE : "+productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
System.out.println("LOCAL : /tmp/"+a.getName());
sftp.get(productionConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
channel.disconnect();
session.disconnect();
File f = new File("/tmp/"+a.getName());
svnUpdater.updateAlgorithmFiles(f);
f.delete();
} }
@ -242,8 +68,15 @@ public class CheckMethodProduction extends CheckMethod{
//System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", //System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
// "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462")); // "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"));
System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org", try
"708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")); {
a.checkMethod("dataminer-ghost-d.dev.d4science.org",
"708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
} catch (Exception e)
{
e.printStackTrace();
}
//Algorithm aa = new Algorithm(); //Algorithm aa = new Algorithm();

View File

@ -2,216 +2,32 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.impl;
import java.io.File; import java.io.File;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager; import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.DMPMClientConfiguratorManager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.jcraft.jsch.Channel;
import com.jcraft.jsch.ChannelSftp;
import com.jcraft.jsch.JSch;
import com.jcraft.jsch.Session;
import com.jcraft.jsch.SftpException;
public class CheckMethodStaging extends CheckMethod{ public class CheckMethodStaging extends CheckMethod{
private Logger logger;
public CheckMethodStaging() public CheckMethodStaging()
{ {
this.logger = LoggerFactory.getLogger(CheckMethodStaging.class); super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration());
} }
@Override
public boolean algoExists(Algorithm algo/*, String env*/) throws Exception{
this.logger.debug("Looking if algo "+algo.getName()+ " exists");
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath());
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))
{
this.logger.debug("Files found");
this.copyFromDmToSVN(file/*,env*/);
this.copyFromDmToSVN(file2/*,env*/);
System.out.println("Files have been copied to SVN");
return true;
}
else
{
this.logger.debug("Files not found");
System.out.println("Algorithm "+algo.getName()+".jar"+ " and "+algo.getName()+"_interface.jar files are not present at the expected path");
return false;
}
}
@Override @Override
public void deleteFiles(Algorithm a/*,String env*/) throws Exception{ protected void copyFromDmToSVN(File a) throws Exception{
JSch jsch = new JSch(); super.copyFromDmToSVN(a, new SVNUpdaterStaging());
Session session = null;
Channel channel = null;
ChannelSftp c = null;
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
System.out.println("checking existing in env: " + stagingConfiguration.getHost());
File file = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", stagingConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
if(doesExist(file.getPath()/*,env*/)&&(doesExist(file2.getPath()/*,env*/))){
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
}
else System.out.println("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
}
@Override
public boolean doesExist(String path/*, String env*/) throws Exception {
this.logger.debug("Looking if file "+path + " exists");
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
boolean success = false;
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
this.logger.debug("Staging configuration host "+stagingConfiguration.getHost());
session = jsch.getSession("root",stagingConfiguration.getHost() );
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
//SftpATTRS is = null;
System.out.println(path);
try {
c.lstat(path);
this.logger.debug("File found");
success = true;
} catch (SftpException e)
{
this.logger.error("File not found",e);
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE)
{
// file doesn't exist
success = false;
}
//success = true; // something else went wrong
}
channel.disconnect();
c.disconnect();
session.disconnect();
this.logger.debug("Operation result "+success);
return success;
}
@Override
public void copyFromDmToSVN(File a/*,String env*/) throws Exception {
JSch jsch = new JSch();
Session session = null;
SVNUpdater svnUpdater = new SVNUpdaterStaging();
Configuration stagingConfiguration = DMPMClientConfiguratorManager.getInstance().getStagingConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", stagingConfiguration.getHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
ChannelSftp sftp = (ChannelSftp) channel;
sftp.cd(stagingConfiguration.getGhostAlgoDirectory());
System.out.println("REMOTE : "+stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName());
System.out.println("LOCAL : /tmp/"+a.getName());
sftp.get(stagingConfiguration.getGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
channel.disconnect();
session.disconnect();
File f = new File("/tmp/"+a.getName());
svnUpdater.updateAlgorithmFiles(f);
f.delete();
} }
@ -258,8 +74,15 @@ public class CheckMethodStaging extends CheckMethod{
//System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org", //System.out.println(a.checkMethod("dataminer-proto-ghost.d4science.org",
// "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462")); // "3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"));
System.out.println(a.checkMethod("dataminer-ghost-d.dev.d4science.org", try
"708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")); {
a.checkMethod("dataminer-ghost-d.dev.d4science.org",
"708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
} catch (Exception e)
{
e.printStackTrace();
}
//Algorithm aa = new Algorithm(); //Algorithm aa = new Algorithm();

View File

@ -8,23 +8,12 @@ public class NotificationHelperProduction extends NotificationHelper{
// private Exception executionException; // private Exception executionException;
public NotificationHelperProduction() {
private String getSubjectHeader() { super ("[DataMinerGhostProductionInstallationRequestReport]");
return "[DataMinerGhostProductionInstallationRequestReport]"; }
}
@Override
public String getSuccessSubject() {
return this.getSubjectHeader()+" is SUCCESS";
}
@Override
public String getFailedSubject() {
return String.format(this.getSubjectHeader()+" is FAILED");
}
} }

View File

@ -8,25 +8,11 @@ public class NotificationHelperStaging extends NotificationHelper {
// private Exception executionException; // private Exception executionException;
public NotificationHelperStaging() {
super ("[DataMinerGhostStagingInstallationRequestReport]");
}
private String getSubjectHeader() {
return "[DataMinerGhostStagingInstallationRequestReport]";
}
@Override
public String getSuccessSubject() {
return this.getSubjectHeader()+" is SUCCESS";
}
@Override
public String getFailedSubject() {
return String.format(this.getSubjectHeader()+" is FAILED");
}
} }

View File

@ -0,0 +1,6 @@
1=ciro.formisano
2=lucio.lelii
3=roberto.cirillo
4=gianpaolo.coro
5=giancarlo.panichi
6=scarponi