This commit is contained in:
Ciro Formisano 2017-12-15 15:46:26 +00:00
parent 85b523e2f2
commit 623ff51624
12 changed files with 377 additions and 237 deletions

View File

@ -15,16 +15,17 @@ import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions.AnsibleException;
import org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions.UndefinedDependenciesException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.AlgorithmException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.DMPMException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.EMailException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException;
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNException;
public abstract class DMPMJob {
@ -144,7 +145,7 @@ public abstract class DMPMJob {
protected abstract void execute ();
private boolean preInstallation (SendMail sm,NotificationHelper nh, File logFile ) throws SVNException, EMailException
private void preInstallation (SendMail sm,NotificationHelper nh, File logFile ) throws GenericException, EMailException,UndefinedDependenciesException
{
this.logger.debug("Checking dependencies...");
@ -155,41 +156,26 @@ public abstract class DMPMJob {
if (!undefinedDependencies.isEmpty())
{
this.logger.debug("Some dependencies are not defined");
String message = "Following dependencies are not defined:\n";
for (String n : undefinedDependencies) {
message += "\n" + n +"\n";
***REMOVED***
this.setStatusInformation(STATUS.FAILED);
String errorMessage = nh.getFailedBody(message+"\n\n"+this.buildInfo());
this.updateLogFile(logFile, errorMessage);
sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", errorMessage);
return false;
***REMOVED***
else return true;
throw new UndefinedDependenciesException(undefinedDependencies);
***REMOVED***
private void installation (SendMail sm,NotificationHelper nh,CheckMethod methodChecker,File logFile ) throws Exception
***REMOVED***
private String installation (SendMail sm,NotificationHelper nh,CheckMethod methodChecker,File logFile ) throws DMPMException
{
this.logger.debug("Installation process started");
methodChecker.deleteFiles(this.algorithm/*, env*/);
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.cluster, false, "root"),logFile);
System.out.println("Return code= "+ret);
this.logger.debug("Return code= "+ret);
if (ret != 0)
{
this.logger.debug("Ansible work failed, return code "+ret);
this.setStatusInformation(STATUS.FAILED);
String errorMessage = nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo();
sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",errorMessage);
***REMOVED***
if (ret != 0) throw new AnsibleException(ret);
else if (ret == 0)
else
{
this.logger.debug("Operation completed");
***REMOVED***this.setStatusInformation(STATUS.PROGRESS);
this.logger.debug("Checking the method...");
try
{
methodChecker.checkMethod(this.configuration.getHost(), SecurityTokenProvider.instance.get());
methodChecker.copyAlgorithms(this.algorithm);
this.logger.debug("Method OK and algo exists");
@ -198,73 +184,63 @@ public abstract class DMPMJob {
boolean algorithmListResult = this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.vREName,this.category, this.algorithm_type,
this.algorithm.getFullname());
this.setStatusInformation(STATUS.COMPLETED);
String bodyResponse = algorithmListResult ? nh.getSuccessBody("\n\n"+this.buildInfo()) : nh.getSuccessBody("\n\n"+this.buildInfo())+"\nWARNING: algorithm list could not be updated on SVN";
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", bodyResponse);
***REMOVED*** catch (SVNCommitException e)
{
this.logger.error("Unable to complete the commit operation: "+e.getMessage());
this.setStatusInformation(STATUS.FAILED);
sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",
nh.getFailedBody(
"\n"+
"Commit operation failed for "+e.getFileName()
+ "the message of the SVN Server is the following:\n"+e.getSvnErrorMessage().getMessage()+"\n\n"
+this.buildInfo()));
***REMOVED*** catch (AlgorithmException ae)
{
this.logger.debug("Operation failed");
this.setStatusInformation(STATUS.FAILED);
sm.sendNotification(nh.getFailedSubject() + " for "+this.algorithm.getName()+ " algorithm",
nh.getFailedBody(
"\n"+
"Installation completed but DataMiner Interface not working correctly or files "
+ this.algorithm.getName() + ".jar and " + this.algorithm.getName()
+ "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
***REMOVED***
return algorithmListResult ?"":"\nWARNING: algorithm list could not be updated on SVN";
***REMOVED***
***REMOVED***
protected void execute(NotificationHelper nh, CheckMethod methodChecker)
{
SendMail sm = new SendMail();
File logFile = new File(this.jobLogs,this.id);
try
{
try {
this.logger.debug("Pre installation operations");
File logFile = new File(this.jobLogs,this.id);
***REMOVED***File logFile = new File(this.jobLogs + File.separator + this.id);
boolean preInstallationResponse = preInstallation(sm, nh, logFile);
this.logger.debug("Pre installation operation completed with result "+preInstallationResponse);
if (preInstallationResponse)
{
preInstallation(sm, nh, logFile);
this.logger.debug("Pre installation operation completed");
this.logger.debug("Installation...");
installation(sm, nh, methodChecker, logFile);
String warning = installation(sm, nh, methodChecker, logFile);
this.logger.debug("Installation completed");
this.logger.debug("Warning message "+warning);
this.setStatusInformation(STATUS.COMPLETED);
String bodyResponse = NotificationHelper.getSuccessBody(warning+"\n\n"+this.buildInfo());
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", bodyResponse);
***REMOVED*** catch (DMPMException dmpme)
{
this.logger.error("Operation failed: "+dmpme.getMessage());
this.logger.error("Exception: ",dmpme);
this.setStatusInformation(STATUS.FAILED);
String errorMessage = "\n"+NotificationHelper.getFailedBody(dmpme.getErrorMessage()+"\n\n"+this.buildInfo());
this.updateLogFile(logFile, errorMessage);
sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", errorMessage);
***REMOVED***
***REMOVED*** catch (EMailException eme)
{
this.logger.error("Operation failed and unable to send notification email",eme);
this.logger.error("Unable to send notification email",eme);
***REMOVED***
catch (Exception e) {
e.printStackTrace();
***REMOVED***
***REMOVED***
protected int executeAnsibleWorker(AnsibleWorker worker, File logFile) throws IOException, InterruptedException, SVNException{
protected int executeAnsibleWorker(AnsibleWorker worker, File logFile) throws GenericException
{
try
{
FileOutputStream fos = new FileOutputStream(logFile, true);
PrintStream ps = new PrintStream(fos);
@ -272,6 +248,12 @@ public abstract class DMPMJob {
***REMOVED*** PrintWriter fos2 = new PrintWriter(m, "UTF-8");
return worker.execute(ps);
***REMOVED*** catch (Exception e)
{
throw new GenericException(e);
***REMOVED***
***REMOVED***
public String buildInfo() {

View File

@ -0,0 +1,25 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.DMPMException;
public class AnsibleException extends DMPMException {
/**
*
*/
private static final long serialVersionUID = 6772009633547404120L;
private int returnCode;
public AnsibleException(int returnCode) {
super ("Ansible work failed");
this.returnCode =returnCode;
***REMOVED***
@Override
public String getErrorMessage() {
return "Installation failed. Return code=" + this.returnCode;
***REMOVED***
***REMOVED***

View File

@ -0,0 +1,29 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service.exceptions;
import java.util.Collection;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.DMPMException;
public class UndefinedDependenciesException extends DMPMException {
private String message;
/**
*
*/
private static final long serialVersionUID = 4504593796352609191L;
public UndefinedDependenciesException(Collection<String> undefinedDependencies) {
super ("Some dependencies are not defined");
this.message = "Following dependencies are not defined:\n";
for (String n : undefinedDependencies) {
message += "\n" + n +"\n";
***REMOVED***
***REMOVED***
@Override
public String getErrorMessage() {
return this.message;
***REMOVED***
***REMOVED***

View File

@ -14,6 +14,7 @@ import java.util.Properties;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.AlgorithmException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException;
***REMOVED***
***REMOVED***
@ -41,6 +42,7 @@ public abstract class CheckMethod {
public CheckMethod(Configuration configuration)
{
this.logger = LoggerFactory.getLogger(CheckMethod.class);
this.configuration = configuration;
sshConfig = new java.util.Properties();
sshConfig.put("StrictHostKeyChecking", "no");
***REMOVED***
@ -124,20 +126,39 @@ public abstract class CheckMethod {
public void copyAlgorithms(Algorithm algo/*, String env*/) throws SVNCommitException, Exception{
public void copyAlgorithms(Algorithm algo/*, String env*/) throws SVNCommitException, GenericException, AlgorithmException
{
this.logger.debug("Looking if algo "+algo.getName()+ " exists");
File file = new File(this.configuration.getGhostAlgoDirectory()+"/"+algo.getName()+".jar");
File file2 = new File(this.configuration.getGhostAlgoDirectory()+"/"+algo.getName()+"_interface.jar");
this.logger.debug("Looking for files "+file.getPath()+ " "+file.getPath());
boolean fileExists = false;
try
{
fileExists = (this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/));
***REMOVED*** catch (Exception e)
{
throw new GenericException(e);
***REMOVED***
if ((this.doesExist(file.getPath()/*,env*/)) && (this.doesExist(file2.getPath()/*,env*/)))
if (fileExists)
{
try
{
this.logger.debug("Files found");
this.copyFromDmToSVN(file/*,env*/);
this.copyFromDmToSVN(file2/*,env*/);
this.logger.debug("Files have been copied to SVN");
***REMOVED*** catch (Exception e)
{
throw new GenericException(e);
***REMOVED***
***REMOVED***
else
{
@ -150,26 +171,28 @@ public abstract class CheckMethod {
public void deleteFiles(Algorithm a/*,String env*/) throws Exception
public void deleteFiles(Algorithm a/*,String env*/) throws GenericException
{
try
{
Session session = generateSession();
System.out.println("checking existing in env: " + this.configuration.getHost());
this.logger.debug("checking existing in env: " + this.configuration.getHost());
File file = new File(this.configuration.getGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(this.configuration.getGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
this.logger.debug("First file is located to: "+file.getPath());
this.logger.debug("Second file is located to: "+file2.getPath());
System.out.println("session created.");
this.logger.debug("session created.");
session.setConfig(this.sshConfig);
session.connect();
Channel channel = session.openChannel(SFTP_PROTOCOL);
channel.connect();
System.out.println("shell channel connected....");
this.logger.debug("shell channel connected....");
ChannelSftp c = (ChannelSftp) channel;
@ -177,12 +200,18 @@ public abstract class CheckMethod {
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
this.logger.debug("Both the files have been deleted");
***REMOVED***
else System.out.println("Files not found");
else this.logger.debug("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
***REMOVED*** catch (Exception e)
{
throw new GenericException(e);
***REMOVED***
***REMOVED***

View File

@ -36,13 +36,13 @@ public abstract class NotificationHelper {
***REMOVED***
public String getSuccessBody(String info) {
public static String getSuccessBody(String info) {
String message = String.format("The installation of the algorithm is completed successfully.");
message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info;
return message;
***REMOVED***
public String getFailedBody(String message) {
public static String getFailedBody(String message) {
String body = String.format("An error occurred while deploying your algorithm");
body+= "\n\nHere are the error details:\n\n" + message;
return body;

View File

@ -24,6 +24,7 @@ import java.util.TimeZone;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.GenericException;
import org.gcube.dataanalysis.dataminer.poolmanager.util.exception.SVNCommitException;
***REMOVED***
***REMOVED***
@ -162,9 +163,7 @@ public abstract class SVNUpdater {
if (errorMessage != null)
{
this.logger.error("Operation failed: "+errorMessage.getFullMessage());
SVNCommitException exception = new SVNCommitException(errorMessage);
exception.setFileName(fileName);
throw exception;
throw new SVNCommitException(errorMessage,fileName);
***REMOVED***
this.logger.debug("Operation completed");
@ -195,9 +194,8 @@ public abstract class SVNUpdater {
if (errorMessage != null)
{
this.logger.error("Operation failed: "+errorMessage.getFullMessage());
SVNCommitException exception = new SVNCommitException(errorMessage);
exception.setFileName(fileName+" to be updated");
throw exception;
throw new SVNCommitException(errorMessage,fileName+" to be updated");
***REMOVED***
this.logger.debug("Operation completed");
@ -318,7 +316,10 @@ public abstract class SVNUpdater {
***REMOVED***
public Collection<String> getUndefinedDependencies(String file, Collection<Dependency> deps) throws SVNException {
public Collection<String> getUndefinedDependencies(String file, Collection<Dependency> deps) throws GenericException
{
try
{
***REMOVED*** SendMail sm = new SendMail();
***REMOVED*** NotificationHelper nh = new NotificationHelper();
@ -332,7 +333,7 @@ public abstract class SVNUpdater {
***REMOVED***
System.out.println("Checking dependencies list: " + file);
this.logger.debug("Checking dependencies list: " + file);
List<String> validDependencies = new LinkedList<String>();
@ -346,7 +347,7 @@ public abstract class SVNUpdater {
validDependencies.add(l.trim());
***REMOVED******REMOVED***
System.out.println("Valid dependencies are: "+validDependencies);
this.logger.debug("Valid dependencies are: "+validDependencies);
for(Dependency d: deps){
String depName = d.getName();
if(!validDependencies.contains(depName)){
@ -403,6 +404,12 @@ public abstract class SVNUpdater {
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** return check;
***REMOVED*** catch (SVNException e)
{
throw new GenericException(e);
***REMOVED***
***REMOVED***

View File

@ -144,6 +144,7 @@ public class SendMail {
public String sendPostRequest(String endpoint, String requestParameters) {
this.logger.debug("Sending post request");
***REMOVED*** Build parameter string
String data = requestParameters;
try {
@ -169,13 +170,17 @@ public class SendMail {
writer.close();
reader.close();
this.logger.debug("Operation completed");
String response = answer.toString();
this.logger.debug("Response "+response);
***REMOVED*** Output the response
return answer.toString();
return response;
***REMOVED*** catch (MalformedURLException ex) {
ex.printStackTrace();
this.logger.error("Invalid URL",ex);
***REMOVED*** catch (IOException ex) {
ex.printStackTrace();
this.logger.error("Error in the IO process",ex);
***REMOVED***
return null;
***REMOVED***

View File

@ -1,25 +1,34 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
public class AlgorithmException extends Exception {
public class AlgorithmException extends DMPMException{
/**
*
*/
private static final long serialVersionUID = -5678597187512954288L;
private String algorithmName;
public AlgorithmException ()
public AlgorithmException (String algorithmName)
{
super ();
super ("Algorithm exception");
this.algorithmName = algorithmName;
***REMOVED***
public AlgorithmException (String message)
public AlgorithmException (String algorithmName, Throwable cause)
{
super (message);
super ("Algorithm exception", cause);
this.algorithmName = algorithmName;
***REMOVED***
public AlgorithmException (String message, Throwable cause)
{
super (message, cause);
@Override
public String getErrorMessage() {
return "Installation completed but DataMiner Interface not working correctly or files "
+ this.algorithmName + ".jar and " + this.algorithmName
+ "_interface.jar not availables at the expected path";
***REMOVED***

View File

@ -0,0 +1,20 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
public abstract class DMPMException extends Exception{
/**
*
*/
private static final long serialVersionUID = 1L;
public DMPMException (String errorMessage)
{
super (errorMessage);
***REMOVED***
public DMPMException(String errorMessage,Throwable cause) {
super (errorMessage,cause);
***REMOVED***
public abstract String getErrorMessage ();
***REMOVED***

View File

@ -3,6 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
public class EMailException extends Exception {
private static final String MESSAGE = "Unable to send email notification";
/**
*
@ -10,10 +11,14 @@ public class EMailException extends Exception {
private static final long serialVersionUID = 1L;
public EMailException() {
super ("Unable to send email notification");
super (MESSAGE);
***REMOVED***
public EMailException(Throwable cause) {
super ("Unable to send email notification",cause);
super (MESSAGE,cause);
***REMOVED***
***REMOVED***

View File

@ -0,0 +1,26 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
public class GenericException extends DMPMException {
/**
*
*/
private static final long serialVersionUID = 6772009633547404120L;
public GenericException(Throwable cause) {
super ("Generic exception",cause);
***REMOVED***
@Override
public String getErrorMessage() {
return this.getCause().getMessage();
***REMOVED***
***REMOVED***

View File

@ -2,7 +2,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util.exception;
import org.tmatesoft.svn.core.SVNErrorMessage;
public class SVNCommitException extends Exception {
public class SVNCommitException extends DMPMException {
/**
*
@ -13,28 +13,31 @@ public class SVNCommitException extends Exception {
private String fileName;
public SVNCommitException(SVNErrorMessage errorMessage) {
public SVNCommitException(SVNErrorMessage errorMessage, String fileName) {
super ("Unable to commit");
this.svnErrorMessage = errorMessage;
this.fileName = fileName;
***REMOVED***
public SVNCommitException(String message,SVNErrorMessage errorMessage) {
public SVNCommitException(String message,SVNErrorMessage errorMessage,String fileName) {
super (message);
this.svnErrorMessage = errorMessage;
this.fileName = fileName;
***REMOVED***
public SVNErrorMessage getSvnErrorMessage() {
return svnErrorMessage;
***REMOVED***
public void setFileName (String fileName)
{
this.fileName = fileName;
***REMOVED***
public String getFileName ()
{
return this.fileName;
@Override
public String getErrorMessage() {
return "Commit operation failed for "+this.fileName
+ "the message of the SVN Server is the following:\n"+this.svnErrorMessage.getMessage();
***REMOVED***