git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@150848 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
9abc3752a2
commit
811107f682
|
|
@ -2,29 +2,27 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
|||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import java.util.Collection;
|
||||
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
|
||||
|
||||
public class StagingJob extends DMPMJob ***REMOVED***
|
||||
|
||||
|
||||
private Algorithm algorithm;
|
||||
private Cluster stagingCluster;
|
||||
private Cluster rProtoCluster;
|
||||
private String rProtoVREName;
|
||||
|
||||
public StagingJob(
|
||||
SVNUpdater svnUpdater,
|
||||
Algorithm algorithm,
|
||||
Cluster stagingCluster,
|
||||
Cluster rProtoCluster,
|
||||
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster,
|
||||
String rProtoVREName) ***REMOVED***
|
||||
super(svnUpdater);
|
||||
this.algorithm = algorithm;
|
||||
|
|
@ -38,11 +36,24 @@ public class StagingJob extends DMPMJob ***REMOVED***
|
|||
ServiceConfiguration a = new ServiceConfiguration();
|
||||
CheckMethod b = new CheckMethod();
|
||||
SendMail sm = new SendMail();
|
||||
|
||||
NotificationHelper nh = new NotificationHelper();
|
||||
|
||||
try ***REMOVED***
|
||||
|
||||
this.svnUpdater.readRPRotoDeps(this.algorithm);
|
||||
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
|
||||
this.svnUpdater.getRProtoDependencyFile(this.algorithm.getLanguage()), this.algorithm.getDependencies());
|
||||
|
||||
if (!undefinedDependencies.isEmpty()) ***REMOVED***
|
||||
|
||||
String message = "Following dependencies are not defined:\n";
|
||||
for (String n : undefinedDependencies) ***REMOVED***
|
||||
message += "\n" + n;
|
||||
***REMOVED***
|
||||
|
||||
sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody(message));
|
||||
return;
|
||||
***REMOVED***
|
||||
|
||||
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root"));
|
||||
|
||||
if (ret == 0) ***REMOVED***
|
||||
|
|
@ -50,19 +61,18 @@ public class StagingJob extends DMPMJob ***REMOVED***
|
|||
if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get())
|
||||
&& (b.algoExists(this.algorithm))) ***REMOVED***
|
||||
|
||||
sm.sendNotification("", "");
|
||||
sm.sendNotification(nh.getSuccessSubject(), nh.getSuccessBody());
|
||||
***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm);
|
||||
|
||||
this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName,
|
||||
this.algorithm.getFullname(), "Proto");
|
||||
|
||||
|
||||
***REMOVED*** int ret2 = this.executeAnsibleWorker(
|
||||
***REMOVED*** createWorker(this.algorithm, this.rProtoCluster,
|
||||
***REMOVED*** false, "gcube"));
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
else sm.sendNotification("", "");
|
||||
***REMOVED*** else
|
||||
sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("Installation failed. Return code=" + ret));
|
||||
|
||||
***REMOVED*** catch (Exception e) ***REMOVED***
|
||||
e.printStackTrace();
|
||||
|
|
|
|||
|
|
@ -9,43 +9,15 @@ import scala.actors.threadpool.Arrays;
|
|||
|
||||
public class NotificationHelper ***REMOVED***
|
||||
|
||||
private Calendar startTime;
|
||||
|
||||
private String taskId;
|
||||
|
||||
private String scope;
|
||||
|
||||
private Exception executionException;
|
||||
|
||||
public NotificationHelper() ***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public void setStartTime(Calendar startTime) ***REMOVED***
|
||||
this.startTime = startTime;
|
||||
***REMOVED***
|
||||
|
||||
public void setTaskId(String taskId) ***REMOVED***
|
||||
this.taskId = taskId;
|
||||
***REMOVED***
|
||||
|
||||
public void setScope(String scope) ***REMOVED***
|
||||
this.scope = scope;
|
||||
***REMOVED***
|
||||
|
||||
private String getSubjectHeader() ***REMOVED***
|
||||
return "[DataMinerGhostInstallationRequestReport]";
|
||||
***REMOVED***
|
||||
|
||||
private String getSpecificVREName() ***REMOVED***
|
||||
if(this.scope!=null) ***REMOVED***
|
||||
String[] parts = this.scope.split("/");
|
||||
if(parts.length>=3) ***REMOVED***
|
||||
return StringUtils.join(Arrays.copyOfRange(parts, 3, parts.length), "/");
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return this.scope;
|
||||
***REMOVED***
|
||||
|
||||
private boolean isError() ***REMOVED***
|
||||
return this.executionException!=null;
|
||||
***REMOVED***
|
||||
|
|
@ -54,45 +26,40 @@ public class NotificationHelper ***REMOVED***
|
|||
this.executionException = executionException;
|
||||
***REMOVED***
|
||||
|
||||
private String getSuccessSubject() ***REMOVED***
|
||||
return String.format("%s Results for your experiment '%s' are ready", this.getSubjectHeader(), this.taskId);
|
||||
public String getSuccessSubject() ***REMOVED***
|
||||
return this.getSubjectHeader()+" is SUCCESS";
|
||||
***REMOVED***
|
||||
|
||||
private String getFailedSubject() ***REMOVED***
|
||||
return String.format("%s An error occurred while executing your experiment '%s'", this.getSubjectHeader(), this.taskId);
|
||||
public String getFailedSubject() ***REMOVED***
|
||||
return String.format(this.getSubjectHeader()+" is FAILED");
|
||||
***REMOVED***
|
||||
|
||||
private String getFormattedStartTime() ***REMOVED***
|
||||
SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z");
|
||||
return sdf.format(this.startTime.getTime());
|
||||
***REMOVED***
|
||||
|
||||
private String getSuccessBody() ***REMOVED***
|
||||
String message = String.format("Your experiment '%s' submitted on %s in the '%s' VRE completed successfully.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName());
|
||||
public String getSuccessBody() ***REMOVED***
|
||||
String message = String.format("The installation of the algorithm in the ghost dataminer is completed successfully.");
|
||||
message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.";
|
||||
return message;
|
||||
***REMOVED***
|
||||
|
||||
private String getFailedBody() ***REMOVED***
|
||||
String message = String.format("An error occurred while executing your experiment '%s' submitted on %s in the '%s' VRE.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName());
|
||||
message+= "\n\nHere are the error details:\n\n" + this.executionException;
|
||||
return message;
|
||||
public String getFailedBody(String message) ***REMOVED***
|
||||
String body = String.format("An error occurred while deploying your algorithm");
|
||||
body+= "\n\nHere are the error details:\n\n" + message;
|
||||
return body;
|
||||
***REMOVED***
|
||||
|
||||
public String getSubject() ***REMOVED***
|
||||
if(this.isError()) ***REMOVED***
|
||||
return this.getFailedSubject();
|
||||
***REMOVED*** else ***REMOVED***
|
||||
return this.getSuccessSubject();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public String getBody() ***REMOVED***
|
||||
if(this.isError()) ***REMOVED***
|
||||
return this.getFailedBody();
|
||||
***REMOVED*** else ***REMOVED***
|
||||
return this.getSuccessBody();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** public String getSubject() ***REMOVED***
|
||||
***REMOVED*** if(this.isError()) ***REMOVED***
|
||||
***REMOVED*** return this.getFailedSubject();
|
||||
***REMOVED*** ***REMOVED*** else ***REMOVED***
|
||||
***REMOVED*** return this.getSuccessSubject();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** public String getBody() ***REMOVED***
|
||||
***REMOVED*** if(this.isError()) ***REMOVED***
|
||||
***REMOVED*** return this.getFailedBody();
|
||||
***REMOVED*** ***REMOVED*** else ***REMOVED***
|
||||
***REMOVED*** return this.getSuccessBody();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -31,117 +31,166 @@ public class SVNUpdater ***REMOVED***
|
|||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
||||
this.svnRepository.setAuthenticationManager(authManager);
|
||||
this.configuration = configuration;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public void updateRPRotoDeps(Algorithm algorithm)***REMOVED***
|
||||
public void updateRPRotoDeps(Algorithm algorithm) ***REMOVED***
|
||||
this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies());
|
||||
this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies());
|
||||
this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
|
||||
***REMOVED***
|
||||
|
||||
public String getRProtoDependencyFile(String language) ***REMOVED***
|
||||
switch (language) ***REMOVED***
|
||||
case "R":
|
||||
return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "R-blackbox":
|
||||
***REMOVED*** return "";
|
||||
***REMOVED*** case "Java":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
case "Knime-Workflow":
|
||||
return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Linux-compiled":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Octave":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Python":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Pre-Installed":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Windows-compiled":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
default:
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public void readRPRotoDeps(Algorithm algorithm) throws SVNException***REMOVED***
|
||||
if (algorithm.getLanguage().equals("R"))***REMOVED***
|
||||
public String getProdDependencyFile(String language) ***REMOVED***
|
||||
switch (language) ***REMOVED***
|
||||
case "R":
|
||||
return this.configuration.getSVNRProdCRANDepsList();
|
||||
***REMOVED*** case "R-blackbox":
|
||||
***REMOVED*** return "";
|
||||
***REMOVED*** case "Java":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Knime-Workflow":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Linux-compiled":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Octave":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Python":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Pre-Installed":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
***REMOVED*** case "Windows-compiled":
|
||||
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
|
||||
default:
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public void readRPRotoDeps(Algorithm algorithm) throws SVNException ***REMOVED***
|
||||
if (algorithm.getLanguage().equals("R")) ***REMOVED***
|
||||
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("R-blackbox"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("R-blackbox")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Java"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Java")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Knime-Workflow"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Knime-Workflow")) ***REMOVED***
|
||||
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Linux-compiled"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Linux-compiled")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Octave"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Octave")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Python"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Python")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Windows-compiled"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Windows-compiled")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Pre-Installed"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Pre-Installed")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public void readProdDeps(Algorithm algorithm) throws SVNException***REMOVED***
|
||||
if (algorithm.getLanguage().equals("R"))***REMOVED***
|
||||
public void readProdDeps(Algorithm algorithm) throws SVNException ***REMOVED***
|
||||
if (algorithm.getLanguage().equals("R")) ***REMOVED***
|
||||
this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("R-blackbox"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("R-blackbox")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Java"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Java")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Knime-Workflow"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Knime-Workflow")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Linux-compiled"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Linux-compiled")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Octave"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Octave")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Python"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Python")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Windows-compiled"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Windows-compiled")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Pre-Installed"))***REMOVED***
|
||||
if (algorithm.getLanguage().equals("Pre-Installed")) ***REMOVED***
|
||||
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public void updateProdDeps(Algorithm algorithm)***REMOVED***
|
||||
public void updateProdDeps(Algorithm algorithm) ***REMOVED***
|
||||
this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
|
||||
this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
|
||||
this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
|
||||
public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) ***REMOVED***
|
||||
this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
|
||||
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) ***REMOVED***
|
||||
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
|
||||
public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env) ***REMOVED***
|
||||
try ***REMOVED***
|
||||
System.out.println("Updating algorithm list: "+ file);
|
||||
System.out.println("Updating algorithm list: " + file);
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
|
||||
***REMOVED***check if the algorithm is already in the list (match the class name)
|
||||
for(String l: lines)***REMOVED***
|
||||
if(l.contains(algorithm.getClazz()))***REMOVED***
|
||||
***REMOVED*** check if the algorithm is already in the list (match the class
|
||||
***REMOVED*** name)
|
||||
for (String l : lines) ***REMOVED***
|
||||
if (l.contains(algorithm.getClazz())) ***REMOVED***
|
||||
System.out.println("Not updating algorithm list beacuse already present");
|
||||
return;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***the algorithm is not in the list. Add it
|
||||
***REMOVED*** the algorithm is not in the list. Add it
|
||||
List<String> newContent = new LinkedList<>(Arrays.asList(lines));
|
||||
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env));
|
||||
***REMOVED***Collections.sort(newContent);
|
||||
|
||||
***REMOVED*** Collections.sort(newContent);
|
||||
|
||||
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
||||
|
||||
|
|
@ -159,54 +208,131 @@ public class SVNUpdater ***REMOVED***
|
|||
***REMOVED***
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
commitEditor.applyTextDelta(file,md5(originalContents));
|
||||
commitEditor.applyTextDelta(file, md5(originalContents));
|
||||
|
||||
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
||||
new ByteArrayInputStream(bytes), commitEditor, true);
|
||||
commitEditor.closeFile(file, checksum);
|
||||
commitEditor.closeEdit();
|
||||
|
||||
|
||||
***REMOVED***catch(Exception ex)***REMOVED***
|
||||
***REMOVED*** catch (Exception ex) ***REMOVED***
|
||||
ex.printStackTrace();
|
||||
***REMOVED***
|
||||
|
||||
finally ***REMOVED***
|
||||
svnRepository.closeSession();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env)***REMOVED***
|
||||
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env) ***REMOVED***
|
||||
StringBuffer sb = new StringBuffer("| ");
|
||||
sb.append(algorithm.getName() +" | ");
|
||||
sb.append(algorithm.getName() + " | ");
|
||||
sb.append(algorithm.getFullname() + " | ");
|
||||
sb.append(algorithm.getCategory() + " | ");
|
||||
sb.append(env + " | ");
|
||||
sb.append("<notextile>./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
|
||||
sb.append("<notextile>./addAlgorithm.sh " + algorithm.getName() + " " + algorithm.getCategory() + " "
|
||||
+ algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N "
|
||||
+ algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
|
||||
sb.append("none |");
|
||||
return sb.toString();
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public Collection<String> getUndefinedDependencies(String file, Collection<Dependency> deps) throws SVNException ***REMOVED***
|
||||
***REMOVED*** SendMail sm = new SendMail();
|
||||
***REMOVED*** NotificationHelper nh = new NotificationHelper();
|
||||
|
||||
|
||||
System.out.println("Checking dependencies list: " + file);
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
List<String> validDependencies = Arrays.asList(lines);
|
||||
|
||||
|
||||
|
||||
List<String> undefined = new LinkedList<String>();
|
||||
|
||||
|
||||
|
||||
for(Dependency d: deps)***REMOVED***
|
||||
String depName = d.getName();
|
||||
if(!validDependencies.contains(depName))***REMOVED***
|
||||
undefined.add(depName);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
return undefined;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** for (String a : lines) ***REMOVED***
|
||||
***REMOVED*** for (String b : ldep) ***REMOVED***
|
||||
***REMOVED*** if (b.equals(a)) ***REMOVED***
|
||||
***REMOVED*** System.out.println("The following dependency is correctly written: " + b);
|
||||
***REMOVED*** ***REMOVED*** else
|
||||
***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** boolean check = false;
|
||||
***REMOVED*** try ***REMOVED***
|
||||
***REMOVED*** System.out.println("Checking dependencies list: " + file);
|
||||
***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
***REMOVED*** svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
***REMOVED***
|
||||
***REMOVED*** ***REMOVED*** if(deps.isEmpty())***REMOVED***
|
||||
***REMOVED*** ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody());
|
||||
***REMOVED*** ***REMOVED*** Exception e = new Exception("No dependency specified for this
|
||||
***REMOVED*** ***REMOVED*** algorithm");
|
||||
***REMOVED*** ***REMOVED*** throw e;
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) ***REMOVED***
|
||||
***REMOVED*** List<String> ldep = new LinkedList<>();
|
||||
***REMOVED*** for (Dependency d : deps) ***REMOVED***
|
||||
***REMOVED*** ldep.add(d.getName());
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** for (String a : lines) ***REMOVED***
|
||||
***REMOVED*** for (String b : ldep) ***REMOVED***
|
||||
***REMOVED*** if (b.equals(a)) ***REMOVED***
|
||||
***REMOVED*** System.out.println("The following dependency is correctly written: " + b);
|
||||
***REMOVED*** check = true;
|
||||
***REMOVED*** ***REMOVED*** else
|
||||
***REMOVED*** check = false;
|
||||
***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED*** catch (Exception a) ***REMOVED***
|
||||
***REMOVED*** a.getMessage();
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** return check;
|
||||
|
||||
***REMOVED***
|
||||
|
||||
public boolean checkIfAvaialable(String file, Collection<Dependency> deps) throws SVNException ***REMOVED***
|
||||
SendMail sm = new SendMail();
|
||||
NotificationHelper nh = new NotificationHelper();
|
||||
|
||||
boolean check = false;
|
||||
try***REMOVED***
|
||||
try ***REMOVED***
|
||||
System.out.println("Checking dependencies list: " + file);
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
|
||||
if(deps.isEmpty())***REMOVED***
|
||||
Exception e = new Exception("No dependency specified for this algorithm");
|
||||
throw e;
|
||||
***REMOVED*** if(deps.isEmpty())***REMOVED***
|
||||
***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody());
|
||||
***REMOVED*** Exception e = new Exception("No dependency specified for this
|
||||
***REMOVED*** algorithm");
|
||||
***REMOVED*** throw e;
|
||||
***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
|
||||
|
||||
else if (!deps.isEmpty()) ***REMOVED***
|
||||
***REMOVED*** else if (!deps.isEmpty()) ***REMOVED***
|
||||
List<String> ldep = new LinkedList<>();
|
||||
for (Dependency d : deps) ***REMOVED***
|
||||
ldep.add(d.getName());
|
||||
|
|
@ -214,16 +340,15 @@ public class SVNUpdater ***REMOVED***
|
|||
for (String a : lines) ***REMOVED***
|
||||
for (String b : ldep) ***REMOVED***
|
||||
if (b.equals(a)) ***REMOVED***
|
||||
System.out.println("The following dependency is correctly written: "+b);
|
||||
System.out.println("The following dependency is correctly written: " + b);
|
||||
check = true;
|
||||
***REMOVED*** else
|
||||
check = false;
|
||||
Exception e = new Exception("One or more of the following dependencies are not correctly written: "+b);
|
||||
throw e;
|
||||
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***catch (Exception a)***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** catch (Exception a) ***REMOVED***
|
||||
a.getMessage();
|
||||
***REMOVED***
|
||||
|
||||
|
|
@ -231,21 +356,16 @@ public class SVNUpdater ***REMOVED***
|
|||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public void updateSVN(String file, Collection<Dependency> deps) ***REMOVED***
|
||||
try ***REMOVED***
|
||||
System.out.println("Updating dependencies list: "+ file);
|
||||
System.out.println("Updating dependencies list: " + file);
|
||||
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
|
||||
List<String> ldep = new LinkedList<>();
|
||||
for(Dependency d: deps)***REMOVED***
|
||||
for (Dependency d : deps) ***REMOVED***
|
||||
ldep.add(d.getName());
|
||||
***REMOVED***
|
||||
List<String> aa = this.checkMatch(lines, ldep);
|
||||
|
|
@ -267,48 +387,44 @@ public class SVNUpdater ***REMOVED***
|
|||
***REMOVED***
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
commitEditor.applyTextDelta(file,md5(originalContents));
|
||||
commitEditor.applyTextDelta(file, md5(originalContents));
|
||||
|
||||
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
||||
new ByteArrayInputStream(bytes), commitEditor, true);
|
||||
commitEditor.closeFile(file, checksum);
|
||||
commitEditor.closeEdit();
|
||||
|
||||
|
||||
***REMOVED***catch(Exception ex)***REMOVED***
|
||||
***REMOVED*** catch (Exception ex) ***REMOVED***
|
||||
ex.printStackTrace();
|
||||
***REMOVED***
|
||||
|
||||
finally ***REMOVED***
|
||||
svnRepository.closeSession();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
||||
public static String md5(byte[] contents) ***REMOVED***
|
||||
final byte[] tmp = new byte[1024];
|
||||
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5");
|
||||
final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents),
|
||||
"md5");
|
||||
try ***REMOVED***
|
||||
while (checksumStream.read(tmp) > 0) ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return checksumStream.getDigest();
|
||||
***REMOVED*** catch (IOException e) ***REMOVED***
|
||||
***REMOVED***never happens
|
||||
***REMOVED*** never happens
|
||||
e.printStackTrace();
|
||||
return null;
|
||||
***REMOVED***
|
||||
***REMOVED*** finally ***REMOVED***
|
||||
SVNFileUtil.closeFile(checksumStream);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public List<String> checkMatch(String[] lines, List<String> ls) ***REMOVED***
|
||||
Set<String> ss = new HashSet<String>(ls);
|
||||
ss.addAll(Arrays.asList(lines));
|
||||
return new ArrayList<>(ss);
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -26,81 +26,70 @@ public class SendMail extends StandardLocalInfraAlgorithm ***REMOVED***
|
|||
***REMOVED*** TODO Auto-generated constructor stub
|
||||
***REMOVED***
|
||||
|
||||
public void sendNotification(String subject, String body) throws Exception ***REMOVED***
|
||||
public void sendNotification(String subject, String body) throws Exception ***REMOVED***
|
||||
|
||||
AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope "+ScopeProvider.instance.get());
|
||||
AnalysisLogger.getLogger()
|
||||
.debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get());
|
||||
|
||||
String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get());
|
||||
|
||||
if (!serviceAddress.endsWith("/"))
|
||||
serviceAddress = serviceAddress+"/";
|
||||
serviceAddress = serviceAddress + "/";
|
||||
|
||||
String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" + SecurityTokenProvider.instance.get();
|
||||
requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error
|
||||
String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token="
|
||||
+ SecurityTokenProvider.instance.get();
|
||||
requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", "");
|
||||
|
||||
AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage);
|
||||
|
||||
|
||||
***REMOVED*** put the sender, the recipients, subject and body of the mail here
|
||||
subject=URLEncoder.encode(subject,"UTF-8");
|
||||
body=URLEncoder.encode(body,"UTF-8");
|
||||
String requestParameters = "&sender=dataminer&recipients="+this.getAdmins()+"&subject="+subject+"&body="+body;
|
||||
|
||||
subject = URLEncoder.encode(subject, "UTF-8");
|
||||
body = URLEncoder.encode(body, "UTF-8");
|
||||
String requestParameters = "&sender=dataminer&recipients=" + /*this.getAdmins()*/this.username(SecurityTokenProvider.instance.get()) + "&subject=" + subject + "&body="
|
||||
+ body;
|
||||
|
||||
String response = HttpRequest.sendPostRequest(requestForMessage, requestParameters);
|
||||
AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK ");
|
||||
|
||||
if (response==null)***REMOVED***
|
||||
if (response == null) ***REMOVED***
|
||||
Exception e = new Exception("Error in email sending response");
|
||||
throw e;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED*** public void notifySubmitter(String a, String b) throws Exception ***REMOVED***
|
||||
***REMOVED*** NotificationHelper nh = new NotificationHelper();
|
||||
***REMOVED*** super.sendNotification(nh.getSubject(),
|
||||
***REMOVED*** nh.getBody());
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** public void notifySubmitter(String a, String b) throws Exception ***REMOVED***
|
||||
***REMOVED*** NotificationHelper nh = new NotificationHelper();
|
||||
***REMOVED*** super.sendNotification(nh.getSubject(),
|
||||
***REMOVED*** nh.getBody());
|
||||
***REMOVED*** ***REMOVED***
|
||||
|
||||
|
||||
|
||||
public String username(String token) throws ObjectNotFound, Exception***REMOVED***
|
||||
public String username(String token) throws ObjectNotFound, Exception ***REMOVED***
|
||||
AuthorizationEntry entry = authorizationService().get(token);
|
||||
System.out.println(entry.getClientInfo().getId());
|
||||
return entry.getClientInfo().getId();
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public String retrieveAdminRole() throws Exception***REMOVED***
|
||||
public String retrieveAdminRole() throws Exception ***REMOVED***
|
||||
String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get());
|
||||
|
||||
if (!serviceAddress.endsWith("/"))
|
||||
serviceAddress = serviceAddress+"/";
|
||||
serviceAddress = serviceAddress + "/";
|
||||
|
||||
String requestForMessage = serviceAddress + "2/users/get-usernames-by-global-role";
|
||||
requestForMessage = requestForMessage.replace("https:***REMOVED***", "http:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error
|
||||
String requestParameters = "role-name=Administrator"+"&gcube-token=" + SecurityTokenProvider.instance.get();
|
||||
requestForMessage = requestForMessage.replace("https:***REMOVED***", "http:***REMOVED***").replace(":80", "");
|
||||
String requestParameters = "role-name=Administrator" + "&gcube-token=" + SecurityTokenProvider.instance.get();
|
||||
|
||||
String response = HttpRequest.sendGetRequest(requestForMessage,requestParameters);
|
||||
String response = HttpRequest.sendGetRequest(requestForMessage, requestParameters);
|
||||
System.out.println(response.toString());
|
||||
|
||||
if (response==null)***REMOVED***
|
||||
if (response == null) ***REMOVED***
|
||||
Exception e = new Exception("Error in querying the recipient");
|
||||
throw e;
|
||||
***REMOVED***
|
||||
return response;
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public String getAdmins() throws Exception ***REMOVED***
|
||||
List<String> s = new LinkedList<String>();
|
||||
JSONObject obj = new JSONObject(this.retrieveAdminRole());
|
||||
|
|
@ -114,45 +103,33 @@ public void sendNotification(String subject, String body) throws Exception ***RE
|
|||
s.add(this.username(SecurityTokenProvider.instance.get()));
|
||||
***REMOVED***
|
||||
|
||||
System.out.println(s);
|
||||
return s.toString();
|
||||
return s.toString().replace("[", "").replace("]", "");
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
public String getDescription() ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
public void init() throws Exception ***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
protected void process() throws Exception ***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
protected void setInputParameters() ***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
***REMOVED***
|
||||
public void shutdown() ***REMOVED***
|
||||
***REMOVED***
|
||||
|
|
@ -160,20 +137,19 @@ public void sendNotification(String subject, String body) throws Exception ***RE
|
|||
***REMOVED***
|
||||
|
||||
public static void main(String[] args) throws Exception ***REMOVED***
|
||||
***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
***REMOVED*** SecurityTokenProvider.instance.set("***REMOVED***");
|
||||
|
||||
***REMOVED***
|
||||
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
SecurityTokenProvider.instance.set("***REMOVED***");
|
||||
|
||||
|
||||
***REMOVED***ScopeProvider.instance.set("/gcube/devNext");
|
||||
***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***");
|
||||
|
||||
NotificationHelper nh = new NotificationHelper();
|
||||
|
||||
SendMail sm = new SendMail();
|
||||
***REMOVED***sm.username(SecurityTokenProvider.instance.get());
|
||||
***REMOVED***sm.retrieveAdminRole();
|
||||
sm.getAdmins();
|
||||
***REMOVED***sm.sendNotification("test", "test");
|
||||
sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("test failed"));
|
||||
***REMOVED*** sm.username(SecurityTokenProvider.instance.get());
|
||||
***REMOVED*** sm.retrieveAdminRole();
|
||||
***REMOVED***System.out.println(sm.getAdmins());
|
||||
***REMOVED*** sm.sendNotification("test", "test");
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ public class SVNTests ***REMOVED***
|
|||
|
||||
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ngalante/workspace/dataminer-pool-manager/src/main/resources/service.properties"));
|
||||
|
||||
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0");
|
||||
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/R0ExYjFPVFBrOUlNdEhrTUQyQlZjbUJuQ20rbGFrZ0pHbWJQNStIS0N6Yz0");
|
||||
***REMOVED***algo.setClazz(algo.getClazz() + "TEST");
|
||||
|
||||
***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto");
|
||||
|
|
|
|||
Loading…
Reference in New Issue