diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index 85e857f..fd18555 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -2,29 +2,27 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; import static org.gcube.common.authorization.client.Constants.authorizationService; +import java.util.Collection; + import org.gcube.common.authorization.library.AuthorizationEntry; ***REMOVED*** ***REMOVED*** ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; +import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration; public class StagingJob extends DMPMJob ***REMOVED*** - - + private Algorithm algorithm; private Cluster stagingCluster; private Cluster rProtoCluster; private String rProtoVREName; - public StagingJob( - SVNUpdater svnUpdater, - Algorithm algorithm, - Cluster stagingCluster, - Cluster rProtoCluster, + public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster, String rProtoVREName) ***REMOVED*** super(svnUpdater); this.algorithm = algorithm; @@ -38,31 +36,43 @@ public class StagingJob extends DMPMJob ***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); CheckMethod b = new CheckMethod(); SendMail sm = new SendMail(); - + NotificationHelper nh = new NotificationHelper(); try ***REMOVED*** - this.svnUpdater.readRPRotoDeps(this.algorithm); - int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); - - if (ret == 0) ***REMOVED*** + Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies( + this.svnUpdater.getRProtoDependencyFile(this.algorithm.getLanguage()), this.algorithm.getDependencies()); - if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get()) - && (b.algoExists(this.algorithm))) ***REMOVED*** + if (!undefinedDependencies.isEmpty()) ***REMOVED*** - sm.sendNotification("", ""); - ***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm); - - this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, - this.algorithm.getFullname(), "Proto"); - - - ***REMOVED*** int ret2 = this.executeAnsibleWorker( - ***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, - ***REMOVED*** false, "gcube")); - ***REMOVED*** + String message = "Following dependencies are not defined:\n"; + for (String n : undefinedDependencies) ***REMOVED*** + message += "\n" + n; ***REMOVED*** - else sm.sendNotification("", ""); + + sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody(message)); + return; + ***REMOVED*** + + int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); + + if (ret == 0) ***REMOVED*** + + if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get()) + && (b.algoExists(this.algorithm))) ***REMOVED*** + + sm.sendNotification(nh.getSuccessSubject(), nh.getSuccessBody()); + ***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm); + + this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, + this.algorithm.getFullname(), "Proto"); + + ***REMOVED*** int ret2 = this.executeAnsibleWorker( + ***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, + ***REMOVED*** false, "gcube")); + ***REMOVED*** + ***REMOVED*** else + sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("Installation failed. Return code=" + ret)); ***REMOVED*** catch (Exception e) ***REMOVED*** e.printStackTrace(); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java index 20c5078..4e67f48 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -8,43 +8,15 @@ import org.apache.commons.lang.StringUtils; import scala.actors.threadpool.Arrays; public class NotificationHelper ***REMOVED*** - - private Calendar startTime; - - private String taskId; - - private String scope; private Exception executionException; public NotificationHelper() ***REMOVED*** ***REMOVED*** - public void setStartTime(Calendar startTime) ***REMOVED*** - this.startTime = startTime; - ***REMOVED*** - - public void setTaskId(String taskId) ***REMOVED*** - this.taskId = taskId; - ***REMOVED*** - - public void setScope(String scope) ***REMOVED*** - this.scope = scope; - ***REMOVED*** - private String getSubjectHeader() ***REMOVED*** return "[DataMinerGhostInstallationRequestReport]"; ***REMOVED*** - - private String getSpecificVREName() ***REMOVED*** - if(this.scope!=null) ***REMOVED*** - String[] parts = this.scope.split("/"); - if(parts.length>=3) ***REMOVED*** - return StringUtils.join(Arrays.copyOfRange(parts, 3, parts.length), "/"); - ***REMOVED*** - ***REMOVED*** - return this.scope; - ***REMOVED*** private boolean isError() ***REMOVED*** return this.executionException!=null; @@ -54,45 +26,40 @@ public class NotificationHelper ***REMOVED*** this.executionException = executionException; ***REMOVED*** - private String getSuccessSubject() ***REMOVED*** - return String.format("%s Results for your experiment '%s' are ready", this.getSubjectHeader(), this.taskId); + public String getSuccessSubject() ***REMOVED*** + return this.getSubjectHeader()+" is SUCCESS"; ***REMOVED*** - private String getFailedSubject() ***REMOVED*** - return String.format("%s An error occurred while executing your experiment '%s'", this.getSubjectHeader(), this.taskId); + public String getFailedSubject() ***REMOVED*** + return String.format(this.getSubjectHeader()+" is FAILED"); ***REMOVED*** - - private String getFormattedStartTime() ***REMOVED*** - SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); - return sdf.format(this.startTime.getTime()); - ***REMOVED*** - - private String getSuccessBody() ***REMOVED*** - String message = String.format("Your experiment '%s' submitted on %s in the '%s' VRE completed successfully.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName()); + + public String getSuccessBody() ***REMOVED*** + String message = String.format("The installation of the algorithm in the ghost dataminer is completed successfully."); message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface."; return message; ***REMOVED*** - private String getFailedBody() ***REMOVED*** - String message = String.format("An error occurred while executing your experiment '%s' submitted on %s in the '%s' VRE.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName()); - message+= "\n\nHere are the error details:\n\n" + this.executionException; - return message; + public String getFailedBody(String message) ***REMOVED*** + String body = String.format("An error occurred while deploying your algorithm"); + body+= "\n\nHere are the error details:\n\n" + message; + return body; ***REMOVED*** - public String getSubject() ***REMOVED*** - if(this.isError()) ***REMOVED*** - return this.getFailedSubject(); - ***REMOVED*** else ***REMOVED*** - return this.getSuccessSubject(); - ***REMOVED*** - ***REMOVED*** - - public String getBody() ***REMOVED*** - if(this.isError()) ***REMOVED*** - return this.getFailedBody(); - ***REMOVED*** else ***REMOVED*** - return this.getSuccessBody(); - ***REMOVED*** - ***REMOVED*** +***REMOVED*** public String getSubject() ***REMOVED*** +***REMOVED*** if(this.isError()) ***REMOVED*** +***REMOVED*** return this.getFailedSubject(); +***REMOVED*** ***REMOVED*** else ***REMOVED*** +***REMOVED*** return this.getSuccessSubject(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** public String getBody() ***REMOVED*** +***REMOVED*** if(this.isError()) ***REMOVED*** +***REMOVED*** return this.getFailedBody(); +***REMOVED*** ***REMOVED*** else ***REMOVED*** +***REMOVED*** return this.getSuccessBody(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index 0257a77..e653eff 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -23,190 +23,316 @@ import java.util.*; */ public class SVNUpdater ***REMOVED*** - private SVNRepository svnRepository; - private ServiceConfiguration configuration; + private SVNRepository svnRepository; + private ServiceConfiguration configuration; - public SVNUpdater(ServiceConfiguration configuration) throws SVNException ***REMOVED*** - this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository())); - ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); - this.svnRepository.setAuthenticationManager(authManager); - this.configuration = configuration; - ***REMOVED*** + public SVNUpdater(ServiceConfiguration configuration) throws SVNException ***REMOVED*** + this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository())); + ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); + this.svnRepository.setAuthenticationManager(authManager); + this.configuration = configuration; +***REMOVED*** + public void updateRPRotoDeps(Algorithm algorithm) ***REMOVED*** + this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); + this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); + this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); +***REMOVED*** + public String getRProtoDependencyFile(String language) ***REMOVED*** + switch (language) ***REMOVED*** + case "R": + return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "R-blackbox": +***REMOVED*** return ""; +***REMOVED*** case "Java": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); + case "Knime-Workflow": + return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Linux-compiled": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Octave": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Python": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Pre-Installed": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Windows-compiled": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); + default: + ***REMOVED*** + ***REMOVED*** +***REMOVED*** + + + + public String getProdDependencyFile(String language) ***REMOVED*** + switch (language) ***REMOVED*** + case "R": + return this.configuration.getSVNRProdCRANDepsList(); +***REMOVED*** case "R-blackbox": +***REMOVED*** return ""; +***REMOVED*** case "Java": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Knime-Workflow": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Linux-compiled": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Octave": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Python": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Pre-Installed": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Windows-compiled": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); + default: + ***REMOVED*** + ***REMOVED*** +***REMOVED*** - public void updateRPRotoDeps(Algorithm algorithm)***REMOVED*** - this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); - this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); - this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); - ***REMOVED*** - - - - public void readRPRotoDeps(Algorithm algorithm) throws SVNException***REMOVED*** - if (algorithm.getLanguage().equals("R"))***REMOVED*** - this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("R-blackbox"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Java"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Knime-Workflow"))***REMOVED*** - this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Linux-compiled"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Octave"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Python"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Windows-compiled"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Pre-Installed"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - ***REMOVED*** - - - public void readProdDeps(Algorithm algorithm) throws SVNException***REMOVED*** - if (algorithm.getLanguage().equals("R"))***REMOVED*** - this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("R-blackbox"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Java"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Knime-Workflow"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Linux-compiled"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Octave"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Python"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Windows-compiled"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Pre-Installed"))***REMOVED*** - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - ***REMOVED*** - - + + + + + - public void updateProdDeps(Algorithm algorithm)***REMOVED*** - this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies()); - this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies()); - this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies()); - ***REMOVED*** + public void readRPRotoDeps(Algorithm algorithm) throws SVNException ***REMOVED*** + if (algorithm.getLanguage().equals("R")) ***REMOVED*** + this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("R-blackbox")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Java")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Knime-Workflow")) ***REMOVED*** + this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Linux-compiled")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Octave")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Python")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Windows-compiled")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Pre-Installed")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** +***REMOVED*** + public void readProdDeps(Algorithm algorithm) throws SVNException ***REMOVED*** + if (algorithm.getLanguage().equals("R")) ***REMOVED*** + this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("R-blackbox")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Java")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Knime-Workflow")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Linux-compiled")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Octave")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Python")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Windows-compiled")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Pre-Installed")) ***REMOVED*** + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** +***REMOVED*** - public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED*** - this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env); - ***REMOVED*** + public void updateProdDeps(Algorithm algorithm) ***REMOVED*** + this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies()); + this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies()); + this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies()); +***REMOVED*** + public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) ***REMOVED*** + this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env); +***REMOVED*** - public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED*** - this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env); - ***REMOVED*** + public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) ***REMOVED*** + this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env); +***REMOVED*** - public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env)***REMOVED*** - try ***REMOVED*** - System.out.println("Updating algorithm list: "+ file); - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env) ***REMOVED*** + try ***REMOVED*** + System.out.println("Updating algorithm list: " + file); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - ***REMOVED***check if the algorithm is already in the list (match the class name) - for(String l: lines)***REMOVED*** - if(l.contains(algorithm.getClazz()))***REMOVED*** - System.out.println("Not updating algorithm list beacuse already present"); - return; - ***REMOVED*** - ***REMOVED*** + ***REMOVED*** check if the algorithm is already in the list (match the class + ***REMOVED*** name) + for (String l : lines) ***REMOVED*** + if (l.contains(algorithm.getClazz())) ***REMOVED*** + System.out.println("Not updating algorithm list beacuse already present"); + return; + ***REMOVED*** + ***REMOVED*** - ***REMOVED***the algorithm is not in the list. Add it - List newContent = new LinkedList<>(Arrays.asList(lines)); - newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env)); - ***REMOVED***Collections.sort(newContent); + ***REMOVED*** the algorithm is not in the list. Add it + List newContent = new LinkedList<>(Arrays.asList(lines)); + newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env)); + ***REMOVED*** Collections.sort(newContent); + final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); - final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + byte[] originalContents = byteArrayOutputStream.toByteArray(); - byte[] originalContents = byteArrayOutputStream.toByteArray(); + final ISVNEditor commitEditor = svnRepository.getCommitEditor("update algorithm list", null); + commitEditor.openRoot(-1); + commitEditor.openFile(file, -1); - final ISVNEditor commitEditor = svnRepository.getCommitEditor("update algorithm list", null); - commitEditor.openRoot(-1); - commitEditor.openFile(file, -1); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for (String line : newContent) ***REMOVED*** + baos.write(line.getBytes()); + baos.write("\n".getBytes()); + ***REMOVED*** + byte[] bytes = baos.toByteArray(); - for (String line : newContent) ***REMOVED*** - baos.write(line.getBytes()); - baos.write("\n".getBytes()); - ***REMOVED*** - byte[] bytes = baos.toByteArray(); + commitEditor.applyTextDelta(file, md5(originalContents)); - commitEditor.applyTextDelta(file,md5(originalContents)); + final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, + new ByteArrayInputStream(bytes), commitEditor, true); + commitEditor.closeFile(file, checksum); + commitEditor.closeEdit(); - final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, - new ByteArrayInputStream(bytes), commitEditor, true); - commitEditor.closeFile(file, checksum); - commitEditor.closeEdit(); + ***REMOVED*** catch (Exception ex) ***REMOVED*** + ex.printStackTrace(); + ***REMOVED*** + finally ***REMOVED*** + svnRepository.closeSession(); + ***REMOVED*** +***REMOVED*** - ***REMOVED***catch(Exception ex)***REMOVED*** - ex.printStackTrace(); - ***REMOVED*** + public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env) ***REMOVED*** + StringBuffer sb = new StringBuffer("| "); + sb.append(algorithm.getName() + " | "); + sb.append(algorithm.getFullname() + " | "); + sb.append(algorithm.getCategory() + " | "); + sb.append(env + " | "); + sb.append("./addAlgorithm.sh " + algorithm.getName() + " " + algorithm.getCategory() + " " + + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); + sb.append("none |"); + return sb.toString(); +***REMOVED*** + + + public Collection getUndefinedDependencies(String file, Collection deps) throws SVNException ***REMOVED*** +***REMOVED*** SendMail sm = new SendMail(); +***REMOVED*** NotificationHelper nh = new NotificationHelper(); - finally ***REMOVED*** - svnRepository.closeSession(); - ***REMOVED*** ***REMOVED*** + + System.out.println("Checking dependencies list: " + file); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + List validDependencies = Arrays.asList(lines); + + + List undefined = new LinkedList(); + + + + for(Dependency d: deps)***REMOVED*** + String depName = d.getName(); + if(!validDependencies.contains(depName))***REMOVED*** + undefined.add(depName); + ***REMOVED*** + ***REMOVED*** + + return undefined; +***REMOVED*** +***REMOVED*** +***REMOVED*** for (String a : lines) ***REMOVED*** +***REMOVED*** for (String b : ldep) ***REMOVED*** +***REMOVED*** if (b.equals(a)) ***REMOVED*** +***REMOVED*** System.out.println("The following dependency is correctly written: " + b); +***REMOVED*** ***REMOVED*** else +***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** boolean check = false; +***REMOVED*** try ***REMOVED*** +***REMOVED*** System.out.println("Checking dependencies list: " + file); +***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); +***REMOVED*** svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); +***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); +***REMOVED*** +***REMOVED*** ***REMOVED*** if(deps.isEmpty())***REMOVED*** +***REMOVED*** ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); +***REMOVED*** ***REMOVED*** Exception e = new Exception("No dependency specified for this +***REMOVED*** ***REMOVED*** algorithm"); +***REMOVED*** ***REMOVED*** throw e; +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) ***REMOVED*** +***REMOVED*** List ldep = new LinkedList<>(); +***REMOVED*** for (Dependency d : deps) ***REMOVED*** +***REMOVED*** ldep.add(d.getName()); +***REMOVED*** ***REMOVED*** +***REMOVED*** for (String a : lines) ***REMOVED*** +***REMOVED*** for (String b : ldep) ***REMOVED*** +***REMOVED*** if (b.equals(a)) ***REMOVED*** +***REMOVED*** System.out.println("The following dependency is correctly written: " + b); +***REMOVED*** check = true; +***REMOVED*** ***REMOVED*** else +***REMOVED*** check = false; +***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** catch (Exception a) ***REMOVED*** +***REMOVED*** a.getMessage(); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** return check; - public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env)***REMOVED*** - StringBuffer sb = new StringBuffer("| "); - sb.append(algorithm.getName() +" | "); - sb.append(algorithm.getFullname() + " | "); - sb.append(algorithm.getCategory() + " | "); - sb.append(env + " | "); - sb.append("./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); - sb.append("none |"); - return sb.toString(); - ***REMOVED*** +***REMOVED*** - - public boolean checkIfAvaialable(String file, Collection deps) throws SVNException ***REMOVED*** SendMail sm = new SendMail(); NotificationHelper nh = new NotificationHelper(); boolean check = false; - try***REMOVED*** - System.out.println("Checking dependencies list: " + file); - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - - if(deps.isEmpty())***REMOVED*** - Exception e = new Exception("No dependency specified for this algorithm"); - throw e; - ***REMOVED*** - - - else if (!deps.isEmpty()) ***REMOVED*** + try ***REMOVED*** + System.out.println("Checking dependencies list: " + file); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + + ***REMOVED*** if(deps.isEmpty())***REMOVED*** + ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); + ***REMOVED*** Exception e = new Exception("No dependency specified for this + ***REMOVED*** algorithm"); + ***REMOVED*** throw e; + ***REMOVED*** + ***REMOVED*** ***REMOVED*** + + ***REMOVED*** else if (!deps.isEmpty()) ***REMOVED*** List ldep = new LinkedList<>(); for (Dependency d : deps) ***REMOVED*** ldep.add(d.getName()); @@ -214,101 +340,91 @@ public class SVNUpdater ***REMOVED*** for (String a : lines) ***REMOVED*** for (String b : ldep) ***REMOVED*** if (b.equals(a)) ***REMOVED*** - System.out.println("The following dependency is correctly written: "+b); + System.out.println("The following dependency is correctly written: " + b); check = true; ***REMOVED*** else check = false; - Exception e = new Exception("One or more of the following dependencies are not correctly written: "+b); - throw e; + ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED***catch (Exception a)***REMOVED*** + ***REMOVED*** ***REMOVED*** + ***REMOVED*** catch (Exception a) ***REMOVED*** a.getMessage(); ***REMOVED*** - + return check; - -***REMOVED*** - - - - - - - public void updateSVN(String file, Collection deps) ***REMOVED*** - try ***REMOVED*** - System.out.println("Updating dependencies list: "+ file); - - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - - List ldep = new LinkedList<>(); - for(Dependency d: deps)***REMOVED*** - ldep.add(d.getName()); - ***REMOVED*** - List aa = this.checkMatch(lines, ldep); - Collections.sort(aa); - - final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); - - byte[] originalContents = byteArrayOutputStream.toByteArray(); - - final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); - commitEditor.openRoot(-1); - commitEditor.openFile(file, -1); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - for (String line : aa) ***REMOVED*** - baos.write(line.getBytes()); - baos.write("\n".getBytes()); - ***REMOVED*** - byte[] bytes = baos.toByteArray(); - - commitEditor.applyTextDelta(file,md5(originalContents)); - - final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, - new ByteArrayInputStream(bytes), commitEditor, true); - commitEditor.closeFile(file, checksum); - commitEditor.closeEdit(); - - - ***REMOVED***catch(Exception ex)***REMOVED*** - ex.printStackTrace(); - ***REMOVED*** - - finally ***REMOVED*** - svnRepository.closeSession(); - ***REMOVED*** - ***REMOVED*** - - - - - public static String md5(byte[] contents) ***REMOVED*** - final byte[] tmp = new byte[1024]; - final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5"); - try ***REMOVED*** - while (checksumStream.read(tmp) > 0) ***REMOVED*** - ***REMOVED*** - ***REMOVED*** - return checksumStream.getDigest(); - ***REMOVED*** catch (IOException e) ***REMOVED*** - ***REMOVED***never happens - e.printStackTrace(); - return null; - ***REMOVED*** finally ***REMOVED*** - SVNFileUtil.closeFile(checksumStream); - ***REMOVED*** - ***REMOVED*** - - public List checkMatch(String[] lines, List ls) ***REMOVED*** - Set ss = new HashSet(ls); - ss.addAll(Arrays.asList(lines)); - return new ArrayList<>(ss); - ***REMOVED*** - + +***REMOVED*** + + public void updateSVN(String file, Collection deps) ***REMOVED*** + try ***REMOVED*** + System.out.println("Updating dependencies list: " + file); + + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + + List ldep = new LinkedList<>(); + for (Dependency d : deps) ***REMOVED*** + ldep.add(d.getName()); + ***REMOVED*** + List aa = this.checkMatch(lines, ldep); + Collections.sort(aa); + + final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + + byte[] originalContents = byteArrayOutputStream.toByteArray(); + + final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); + commitEditor.openRoot(-1); + commitEditor.openFile(file, -1); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + for (String line : aa) ***REMOVED*** + baos.write(line.getBytes()); + baos.write("\n".getBytes()); + ***REMOVED*** + byte[] bytes = baos.toByteArray(); + + commitEditor.applyTextDelta(file, md5(originalContents)); + + final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, + new ByteArrayInputStream(bytes), commitEditor, true); + commitEditor.closeFile(file, checksum); + commitEditor.closeEdit(); + + ***REMOVED*** catch (Exception ex) ***REMOVED*** + ex.printStackTrace(); + ***REMOVED*** + + finally ***REMOVED*** + svnRepository.closeSession(); + ***REMOVED*** +***REMOVED*** + + public static String md5(byte[] contents) ***REMOVED*** + final byte[] tmp = new byte[1024]; + final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), + "md5"); + try ***REMOVED*** + while (checksumStream.read(tmp) > 0) ***REMOVED*** + ***REMOVED*** + ***REMOVED*** + return checksumStream.getDigest(); + ***REMOVED*** catch (IOException e) ***REMOVED*** + ***REMOVED*** never happens + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** finally ***REMOVED*** + SVNFileUtil.closeFile(checksumStream); + ***REMOVED*** +***REMOVED*** + + public List checkMatch(String[] lines, List ls) ***REMOVED*** + Set ss = new HashSet(ls); + ss.addAll(Arrays.asList(lines)); + return new ArrayList<>(ss); +***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java index 74a6d15..1ce14c1 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -26,80 +26,69 @@ public class SendMail extends StandardLocalInfraAlgorithm ***REMOVED*** ***REMOVED*** TODO Auto-generated constructor stub ***REMOVED*** -public void sendNotification(String subject, String body) throws Exception ***REMOVED*** - - AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope "+ScopeProvider.instance.get()); - + public void sendNotification(String subject, String body) throws Exception ***REMOVED*** + + AnalysisLogger.getLogger() + .debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get()); + String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); - + if (!serviceAddress.endsWith("/")) - serviceAddress = serviceAddress+"/"; - - String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" + SecurityTokenProvider.instance.get(); - requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error + serviceAddress = serviceAddress + "/"; + + String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" + + SecurityTokenProvider.instance.get(); + requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage); - ***REMOVED*** put the sender, the recipients, subject and body of the mail here - subject=URLEncoder.encode(subject,"UTF-8"); - body=URLEncoder.encode(body,"UTF-8"); - String requestParameters = "&sender=dataminer&recipients="+this.getAdmins()+"&subject="+subject+"&body="+body; - - + subject = URLEncoder.encode(subject, "UTF-8"); + body = URLEncoder.encode(body, "UTF-8"); + String requestParameters = "&sender=dataminer&recipients=" + /*this.getAdmins()*/this.username(SecurityTokenProvider.instance.get()) + "&subject=" + subject + "&body=" + + body; + String response = HttpRequest.sendPostRequest(requestForMessage, requestParameters); AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK "); - - if (response==null)***REMOVED*** + + if (response == null) ***REMOVED*** Exception e = new Exception("Error in email sending response"); throw e; ***REMOVED*** ***REMOVED*** - -***REMOVED*** public void notifySubmitter(String a, String b) throws Exception ***REMOVED*** -***REMOVED*** NotificationHelper nh = new NotificationHelper(); -***REMOVED*** super.sendNotification(nh.getSubject(), -***REMOVED*** nh.getBody()); -***REMOVED*** ***REMOVED*** + ***REMOVED*** public void notifySubmitter(String a, String b) throws Exception ***REMOVED*** + ***REMOVED*** NotificationHelper nh = new NotificationHelper(); + ***REMOVED*** super.sendNotification(nh.getSubject(), + ***REMOVED*** nh.getBody()); + ***REMOVED*** ***REMOVED*** - - public String username(String token) throws ObjectNotFound, Exception***REMOVED*** + public String username(String token) throws ObjectNotFound, Exception ***REMOVED*** AuthorizationEntry entry = authorizationService().get(token); System.out.println(entry.getClientInfo().getId()); return entry.getClientInfo().getId(); - ***REMOVED*** - - - - - - - public String retrieveAdminRole() throws Exception***REMOVED*** + public String retrieveAdminRole() throws Exception ***REMOVED*** String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); - - if (!serviceAddress.endsWith("/")) - serviceAddress = serviceAddress+"/"; - - String requestForMessage = serviceAddress + "2/users/get-usernames-by-global-role"; - requestForMessage = requestForMessage.replace("https:***REMOVED***", "http:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error - String requestParameters = "role-name=Administrator"+"&gcube-token=" + SecurityTokenProvider.instance.get(); - String response = HttpRequest.sendGetRequest(requestForMessage,requestParameters); + if (!serviceAddress.endsWith("/")) + serviceAddress = serviceAddress + "/"; + + String requestForMessage = serviceAddress + "2/users/get-usernames-by-global-role"; + requestForMessage = requestForMessage.replace("https:***REMOVED***", "http:***REMOVED***").replace(":80", ""); + String requestParameters = "role-name=Administrator" + "&gcube-token=" + SecurityTokenProvider.instance.get(); + + String response = HttpRequest.sendGetRequest(requestForMessage, requestParameters); System.out.println(response.toString()); - - if (response==null)***REMOVED*** + + if (response == null) ***REMOVED*** Exception e = new Exception("Error in querying the recipient"); throw e; ***REMOVED*** return response; - - + ***REMOVED*** - - public String getAdmins() throws Exception ***REMOVED*** List s = new LinkedList(); @@ -113,67 +102,54 @@ public void sendNotification(String subject, String body) throws Exception ***RE ***REMOVED*** s.add(this.username(SecurityTokenProvider.instance.get())); ***REMOVED*** - - System.out.println(s); - return s.toString(); + + return s.toString().replace("[", "").replace("]", ""); ***REMOVED*** - - - - + ***REMOVED*** public String getDescription() ***REMOVED*** ***REMOVED*** ***REMOVED*** ***REMOVED*** - - ***REMOVED*** public void init() throws Exception ***REMOVED*** ***REMOVED*** - + ***REMOVED*** - - ***REMOVED*** protected void process() throws Exception ***REMOVED*** ***REMOVED*** - + ***REMOVED*** - - ***REMOVED*** protected void setInputParameters() ***REMOVED*** ***REMOVED*** - + ***REMOVED*** - - ***REMOVED*** public void shutdown() ***REMOVED*** ***REMOVED*** - -***REMOVED*** - - public static void main(String[] args) throws Exception ***REMOVED*** -***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); -***REMOVED*** SecurityTokenProvider.instance.set("***REMOVED***"); ***REMOVED*** + + public static void main(String[] args) throws Exception ***REMOVED*** + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); SecurityTokenProvider.instance.set("***REMOVED***"); - - - + + ***REMOVED***ScopeProvider.instance.set("/gcube/devNext"); + ***REMOVED***SecurityTokenProvider.instance.set("***REMOVED***"); + NotificationHelper nh = new NotificationHelper(); - + SendMail sm = new SendMail(); - ***REMOVED***sm.username(SecurityTokenProvider.instance.get()); - ***REMOVED***sm.retrieveAdminRole(); - sm.getAdmins(); - ***REMOVED***sm.sendNotification("test", "test"); + sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("test failed")); + ***REMOVED*** sm.username(SecurityTokenProvider.instance.get()); + ***REMOVED*** sm.retrieveAdminRole(); + ***REMOVED***System.out.println(sm.getAdmins()); + ***REMOVED*** sm.sendNotification("test", "test"); ***REMOVED*** ***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java index 35937ed..8c6bf9e 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -19,7 +19,7 @@ public class SVNTests ***REMOVED*** SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ngalante/workspace/dataminer-pool-manager/src/main/resources/service.properties")); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0"); + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/R0ExYjFPVFBrOUlNdEhrTUQyQlZjbUJuQ20rbGFrZ0pHbWJQNStIS0N6Yz0"); ***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto");