diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index e7a4b2b..5694e52 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -85,7 +85,7 @@ public class RestPoolManager implements PoolManager { ***REMOVED*** ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); ***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); -***REMOVED*** SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); + SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); ***REMOVED*** AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); ***REMOVED*** System.out.println(entry.getContext()); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index bca4fd7..2644182 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -2,29 +2,27 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; import static org.gcube.common.authorization.client.Constants.authorizationService; +import java.util.Collection; + import org.gcube.common.authorization.library.AuthorizationEntry; ***REMOVED*** ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; +import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration; public class StagingJob extends DMPMJob { - - + private Algorithm algorithm; private Cluster stagingCluster; private Cluster rProtoCluster; private String rProtoVREName; - public StagingJob( - SVNUpdater svnUpdater, - Algorithm algorithm, - Cluster stagingCluster, - Cluster rProtoCluster, + public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster, String rProtoVREName) { super(svnUpdater); this.algorithm = algorithm; @@ -38,31 +36,43 @@ public class StagingJob extends DMPMJob { ServiceConfiguration a = new ServiceConfiguration(); CheckMethod b = new CheckMethod(); SendMail sm = new SendMail(); - + NotificationHelper nh = new NotificationHelper(); try { - this.svnUpdater.readRPRotoDeps(this.algorithm); - int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); - - if (ret == 0) { + Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies( + this.svnUpdater.getRProtoDependencyFile(this.algorithm.getLanguage()), this.algorithm.getDependencies()); - if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get()) - && (b.algoExists(this.algorithm))) { + if (!undefinedDependencies.isEmpty()) { - sm.sendNotification("", ""); - ***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm); - - this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, - this.algorithm.getFullname(), "Proto"); - - - ***REMOVED*** int ret2 = this.executeAnsibleWorker( - ***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, - ***REMOVED*** false, "gcube")); - ***REMOVED*** + String message = "Following dependencies are not defined:\n"; + for (String n : undefinedDependencies) { + message += "\n" + n; ***REMOVED*** - else sm.sendNotification("", ""); + + sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody(message)); + return; + ***REMOVED*** + + int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.stagingCluster, false, "root")); + + if (ret == 0) { + + if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get()) + && (b.algoExists(this.algorithm))) { + + sm.sendNotification(nh.getSuccessSubject(), nh.getSuccessBody()); + ***REMOVED*** this.svnUpdater.updateRPRotoDeps(this.algorithm); + + this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, + this.algorithm.getFullname(), "Proto"); + + ***REMOVED*** int ret2 = this.executeAnsibleWorker( + ***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, + ***REMOVED*** false, "gcube")); + ***REMOVED*** + ***REMOVED*** else + sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("Installation failed. Return code=" + ret)); ***REMOVED*** catch (Exception e) { e.printStackTrace(); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java index f13d8b2..1ec0a7c 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -8,43 +8,15 @@ import org.apache.commons.lang.StringUtils; import scala.actors.threadpool.Arrays; public class NotificationHelper { - - private Calendar startTime; - - private String taskId; - - private String scope; private Exception executionException; public NotificationHelper() { ***REMOVED*** - public void setStartTime(Calendar startTime) { - this.startTime = startTime; - ***REMOVED*** - - public void setTaskId(String taskId) { - this.taskId = taskId; - ***REMOVED*** - - public void setScope(String scope) { - this.scope = scope; - ***REMOVED*** - private String getSubjectHeader() { return "[DataMinerGhostInstallationRequestReport]"; ***REMOVED*** - - private String getSpecificVREName() { - if(this.scope!=null) { - String[] parts = this.scope.split("/"); - if(parts.length>=3) { - return StringUtils.join(Arrays.copyOfRange(parts, 3, parts.length), "/"); - ***REMOVED*** - ***REMOVED*** - return this.scope; - ***REMOVED*** private boolean isError() { return this.executionException!=null; @@ -54,45 +26,40 @@ public class NotificationHelper { this.executionException = executionException; ***REMOVED*** - private String getSuccessSubject() { - return String.format("%s Results for your experiment '%s' are ready", this.getSubjectHeader(), this.taskId); + public String getSuccessSubject() { + return this.getSubjectHeader()+" is SUCCESS"; ***REMOVED*** - private String getFailedSubject() { - return String.format("%s An error occurred while executing your experiment '%s'", this.getSubjectHeader(), this.taskId); + public String getFailedSubject() { + return String.format(this.getSubjectHeader()+" is FAILED"); ***REMOVED*** - - private String getFormattedStartTime() { - SimpleDateFormat sdf = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss Z"); - return sdf.format(this.startTime.getTime()); - ***REMOVED*** - - private String getSuccessBody() { - String message = String.format("Your experiment '%s' submitted on %s in the '%s' VRE completed successfully.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName()); + + public String getSuccessBody() { + String message = String.format("The installation of the algorithm in the ghost dataminer is completed successfully."); message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface."; return message; ***REMOVED*** - private String getFailedBody() { - String message = String.format("An error occurred while executing your experiment '%s' submitted on %s in the '%s' VRE.", this.taskId, this.getFormattedStartTime(), this.getSpecificVREName()); - message+= "\n\nHere are the error details:\n\n" + this.executionException; - return message; + public String getFailedBody(String message) { + String body = String.format("An error occurred while deploying your algorithm"); + body+= "\n\nHere are the error details:\n\n" + message; + return body; ***REMOVED*** - public String getSubject() { - if(this.isError()) { - return this.getFailedSubject(); - ***REMOVED*** else { - return this.getSuccessSubject(); - ***REMOVED*** - ***REMOVED*** - - public String getBody() { - if(this.isError()) { - return this.getFailedBody(); - ***REMOVED*** else { - return this.getSuccessBody(); - ***REMOVED*** - ***REMOVED*** +***REMOVED*** public String getSubject() { +***REMOVED*** if(this.isError()) { +***REMOVED*** return this.getFailedSubject(); +***REMOVED*** ***REMOVED*** else { +***REMOVED*** return this.getSuccessSubject(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** public String getBody() { +***REMOVED*** if(this.isError()) { +***REMOVED*** return this.getFailedBody(); +***REMOVED*** ***REMOVED*** else { +***REMOVED*** return this.getSuccessBody(); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index f0d3d96..186d80a 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -23,190 +23,316 @@ import java.util.*; */ public class SVNUpdater { - private SVNRepository svnRepository; - private ServiceConfiguration configuration; + private SVNRepository svnRepository; + private ServiceConfiguration configuration; - public SVNUpdater(ServiceConfiguration configuration) throws SVNException { - this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository())); - ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); - this.svnRepository.setAuthenticationManager(authManager); - this.configuration = configuration; - ***REMOVED*** + public SVNUpdater(ServiceConfiguration configuration) throws SVNException { + this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository())); + ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); + this.svnRepository.setAuthenticationManager(authManager); + this.configuration = configuration; +***REMOVED*** + public void updateRPRotoDeps(Algorithm algorithm) { + this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); + this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); + this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); +***REMOVED*** + public String getRProtoDependencyFile(String language) { + switch (language) { + case "R": + return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "R-blackbox": +***REMOVED*** return ""; +***REMOVED*** case "Java": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); + case "Knime-Workflow": + return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Linux-compiled": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Octave": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Python": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Pre-Installed": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Windows-compiled": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); + default: + return null; + ***REMOVED*** +***REMOVED*** + + + + public String getProdDependencyFile(String language) { + switch (language) { + case "R": + return this.configuration.getSVNRProdCRANDepsList(); +***REMOVED*** case "R-blackbox": +***REMOVED*** return ""; +***REMOVED*** case "Java": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Knime-Workflow": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Linux-compiled": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Octave": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Python": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Pre-Installed": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); +***REMOVED*** case "Windows-compiled": +***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); + default: + return null; + ***REMOVED*** +***REMOVED*** - public void updateRPRotoDeps(Algorithm algorithm){ - this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); - this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); - this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); - ***REMOVED*** - - - - public void readRPRotoDeps(Algorithm algorithm) throws SVNException{ - if (algorithm.getLanguage().equals("R")){ - this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("R-blackbox")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Java")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Knime-Workflow")){ - this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Linux-compiled")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Octave")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Python")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Windows-compiled")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Pre-Installed")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - ***REMOVED*** - - - public void readProdDeps(Algorithm algorithm) throws SVNException{ - if (algorithm.getLanguage().equals("R")){ - this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("R-blackbox")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Java")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Knime-Workflow")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Linux-compiled")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Octave")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Python")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Windows-compiled")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - if (algorithm.getLanguage().equals("Pre-Installed")){ - this.checkIfAvaialable("", algorithm.getDependencies()); - ***REMOVED*** - ***REMOVED*** - - + + + + + - public void updateProdDeps(Algorithm algorithm){ - this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies()); - this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies()); - this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies()); - ***REMOVED*** + public void readRPRotoDeps(Algorithm algorithm) throws SVNException { + if (algorithm.getLanguage().equals("R")) { + this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("R-blackbox")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Java")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Knime-Workflow")) { + this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Linux-compiled")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Octave")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Python")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Windows-compiled")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Pre-Installed")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** +***REMOVED*** + public void readProdDeps(Algorithm algorithm) throws SVNException { + if (algorithm.getLanguage().equals("R")) { + this.checkIfAvaialable(this.configuration.getSVNRProdCRANDepsList(), algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("R-blackbox")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Java")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Knime-Workflow")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Linux-compiled")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Octave")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Python")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Windows-compiled")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** + if (algorithm.getLanguage().equals("Pre-Installed")) { + this.checkIfAvaialable("", algorithm.getDependencies()); + ***REMOVED*** +***REMOVED*** - public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env){ - this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env); - ***REMOVED*** + public void updateProdDeps(Algorithm algorithm) { + this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies()); + this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies()); + this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies()); +***REMOVED*** + public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) { + this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env); +***REMOVED*** - public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env){ - this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env); - ***REMOVED*** + public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) { + this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env); +***REMOVED*** - public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env){ - try { - System.out.println("Updating algorithm list: "+ file); - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env) { + try { + System.out.println("Updating algorithm list: " + file); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - ***REMOVED***check if the algorithm is already in the list (match the class name) - for(String l: lines){ - if(l.contains(algorithm.getClazz())){ - System.out.println("Not updating algorithm list beacuse already present"); - return; - ***REMOVED*** - ***REMOVED*** + ***REMOVED*** check if the algorithm is already in the list (match the class + ***REMOVED*** name) + for (String l : lines) { + if (l.contains(algorithm.getClazz())) { + System.out.println("Not updating algorithm list beacuse already present"); + return; + ***REMOVED*** + ***REMOVED*** - ***REMOVED***the algorithm is not in the list. Add it - List newContent = new LinkedList<>(Arrays.asList(lines)); - newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env)); - ***REMOVED***Collections.sort(newContent); + ***REMOVED*** the algorithm is not in the list. Add it + List newContent = new LinkedList<>(Arrays.asList(lines)); + newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, env)); + ***REMOVED*** Collections.sort(newContent); + final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); - final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + byte[] originalContents = byteArrayOutputStream.toByteArray(); - byte[] originalContents = byteArrayOutputStream.toByteArray(); + final ISVNEditor commitEditor = svnRepository.getCommitEditor("update algorithm list", null); + commitEditor.openRoot(-1); + commitEditor.openFile(file, -1); - final ISVNEditor commitEditor = svnRepository.getCommitEditor("update algorithm list", null); - commitEditor.openRoot(-1); - commitEditor.openFile(file, -1); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); - ByteArrayOutputStream baos = new ByteArrayOutputStream(); + for (String line : newContent) { + baos.write(line.getBytes()); + baos.write("\n".getBytes()); + ***REMOVED*** + byte[] bytes = baos.toByteArray(); - for (String line : newContent) { - baos.write(line.getBytes()); - baos.write("\n".getBytes()); - ***REMOVED*** - byte[] bytes = baos.toByteArray(); + commitEditor.applyTextDelta(file, md5(originalContents)); - commitEditor.applyTextDelta(file,md5(originalContents)); + final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, + new ByteArrayInputStream(bytes), commitEditor, true); + commitEditor.closeFile(file, checksum); + commitEditor.closeEdit(); - final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, - new ByteArrayInputStream(bytes), commitEditor, true); - commitEditor.closeFile(file, checksum); - commitEditor.closeEdit(); + ***REMOVED*** catch (Exception ex) { + ex.printStackTrace(); + ***REMOVED*** + finally { + svnRepository.closeSession(); + ***REMOVED*** +***REMOVED*** - ***REMOVED***catch(Exception ex){ - ex.printStackTrace(); - ***REMOVED*** + public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env) { + StringBuffer sb = new StringBuffer("| "); + sb.append(algorithm.getName() + " | "); + sb.append(algorithm.getFullname() + " | "); + sb.append(algorithm.getCategory() + " | "); + sb.append(env + " | "); + sb.append("./addAlgorithm.sh " + algorithm.getName() + " " + algorithm.getCategory() + " " + + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); + sb.append("none |"); + return sb.toString(); +***REMOVED*** + + + public Collection getUndefinedDependencies(String file, Collection deps) throws SVNException { +***REMOVED*** SendMail sm = new SendMail(); +***REMOVED*** NotificationHelper nh = new NotificationHelper(); - finally { - svnRepository.closeSession(); - ***REMOVED*** ***REMOVED*** + + System.out.println("Checking dependencies list: " + file); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + List validDependencies = Arrays.asList(lines); + + + List undefined = new LinkedList(); + + + + for(Dependency d: deps){ + String depName = d.getName(); + if(!validDependencies.contains(depName)){ + undefined.add(depName); + ***REMOVED*** + ***REMOVED*** + + return undefined; +***REMOVED*** +***REMOVED*** +***REMOVED*** for (String a : lines) { +***REMOVED*** for (String b : ldep) { +***REMOVED*** if (b.equals(a)) { +***REMOVED*** System.out.println("The following dependency is correctly written: " + b); +***REMOVED*** ***REMOVED*** else +***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** boolean check = false; +***REMOVED*** try { +***REMOVED*** System.out.println("Checking dependencies list: " + file); +***REMOVED*** final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); +***REMOVED*** svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); +***REMOVED*** String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); +***REMOVED*** +***REMOVED*** ***REMOVED*** if(deps.isEmpty()){ +***REMOVED*** ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); +***REMOVED*** ***REMOVED*** Exception e = new Exception("No dependency specified for this +***REMOVED*** ***REMOVED*** algorithm"); +***REMOVED*** ***REMOVED*** throw e; +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** ***REMOVED*** else if (!deps.isEmpty()) { +***REMOVED*** List ldep = new LinkedList<>(); +***REMOVED*** for (Dependency d : deps) { +***REMOVED*** ldep.add(d.getName()); +***REMOVED*** ***REMOVED*** +***REMOVED*** for (String a : lines) { +***REMOVED*** for (String b : ldep) { +***REMOVED*** if (b.equals(a)) { +***REMOVED*** System.out.println("The following dependency is correctly written: " + b); +***REMOVED*** check = true; +***REMOVED*** ***REMOVED*** else +***REMOVED*** check = false; +***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** catch (Exception a) { +***REMOVED*** a.getMessage(); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** return check; - public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String env){ - StringBuffer sb = new StringBuffer("| "); - sb.append(algorithm.getName() +" | "); - sb.append(algorithm.getFullname() + " | "); - sb.append(algorithm.getCategory() + " | "); - sb.append(env + " | "); - sb.append("./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); - sb.append("none |"); - return sb.toString(); - ***REMOVED*** +***REMOVED*** - - public boolean checkIfAvaialable(String file, Collection deps) throws SVNException { SendMail sm = new SendMail(); NotificationHelper nh = new NotificationHelper(); boolean check = false; - try{ - System.out.println("Checking dependencies list: " + file); - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - - if(deps.isEmpty()){ - Exception e = new Exception("No dependency specified for this algorithm"); - throw e; - ***REMOVED*** - - - else if (!deps.isEmpty()) { + try { + System.out.println("Checking dependencies list: " + file); + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + + ***REMOVED*** if(deps.isEmpty()){ + ***REMOVED*** sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody()); + ***REMOVED*** Exception e = new Exception("No dependency specified for this + ***REMOVED*** algorithm"); + ***REMOVED*** throw e; + ***REMOVED*** + ***REMOVED*** ***REMOVED*** + + ***REMOVED*** else if (!deps.isEmpty()) { List ldep = new LinkedList<>(); for (Dependency d : deps) { ldep.add(d.getName()); @@ -214,101 +340,91 @@ public class SVNUpdater { for (String a : lines) { for (String b : ldep) { if (b.equals(a)) { - System.out.println("The following dependency is correctly written: "+b); + System.out.println("The following dependency is correctly written: " + b); check = true; ***REMOVED*** else check = false; - Exception e = new Exception("One or more of the following dependencies are not correctly written: "+b); - throw e; + ***REMOVED*** ***REMOVED*** - ***REMOVED*** - ***REMOVED***catch (Exception a){ + ***REMOVED*** ***REMOVED*** + ***REMOVED*** catch (Exception a) { a.getMessage(); ***REMOVED*** - + return check; - -***REMOVED*** - - - - - - - public void updateSVN(String file, Collection deps) { - try { - System.out.println("Updating dependencies list: "+ file); - - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); - String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); - - List ldep = new LinkedList<>(); - for(Dependency d: deps){ - ldep.add(d.getName()); - ***REMOVED*** - List aa = this.checkMatch(lines, ldep); - Collections.sort(aa); - - final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); - - byte[] originalContents = byteArrayOutputStream.toByteArray(); - - final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); - commitEditor.openRoot(-1); - commitEditor.openFile(file, -1); - - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - - for (String line : aa) { - baos.write(line.getBytes()); - baos.write("\n".getBytes()); - ***REMOVED*** - byte[] bytes = baos.toByteArray(); - - commitEditor.applyTextDelta(file,md5(originalContents)); - - final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, - new ByteArrayInputStream(bytes), commitEditor, true); - commitEditor.closeFile(file, checksum); - commitEditor.closeEdit(); - - - ***REMOVED***catch(Exception ex){ - ex.printStackTrace(); - ***REMOVED*** - - finally { - svnRepository.closeSession(); - ***REMOVED*** - ***REMOVED*** - - - - - public static String md5(byte[] contents) { - final byte[] tmp = new byte[1024]; - final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), "md5"); - try { - while (checksumStream.read(tmp) > 0) { - ***REMOVED*** - ***REMOVED*** - return checksumStream.getDigest(); - ***REMOVED*** catch (IOException e) { - ***REMOVED***never happens - e.printStackTrace(); - return null; - ***REMOVED*** finally { - SVNFileUtil.closeFile(checksumStream); - ***REMOVED*** - ***REMOVED*** - - public List checkMatch(String[] lines, List ls) { - Set ss = new HashSet(ls); - ss.addAll(Arrays.asList(lines)); - return new ArrayList<>(ss); - ***REMOVED*** - + +***REMOVED*** + + public void updateSVN(String file, Collection deps) { + try { + System.out.println("Updating dependencies list: " + file); + + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + + List ldep = new LinkedList<>(); + for (Dependency d : deps) { + ldep.add(d.getName()); + ***REMOVED*** + List aa = this.checkMatch(lines, ldep); + Collections.sort(aa); + + final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + + byte[] originalContents = byteArrayOutputStream.toByteArray(); + + final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); + commitEditor.openRoot(-1); + commitEditor.openFile(file, -1); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + for (String line : aa) { + baos.write(line.getBytes()); + baos.write("\n".getBytes()); + ***REMOVED*** + byte[] bytes = baos.toByteArray(); + + commitEditor.applyTextDelta(file, md5(originalContents)); + + final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, + new ByteArrayInputStream(bytes), commitEditor, true); + commitEditor.closeFile(file, checksum); + commitEditor.closeEdit(); + + ***REMOVED*** catch (Exception ex) { + ex.printStackTrace(); + ***REMOVED*** + + finally { + svnRepository.closeSession(); + ***REMOVED*** +***REMOVED*** + + public static String md5(byte[] contents) { + final byte[] tmp = new byte[1024]; + final SVNChecksumInputStream checksumStream = new SVNChecksumInputStream(new ByteArrayInputStream(contents), + "md5"); + try { + while (checksumStream.read(tmp) > 0) { + ***REMOVED*** + ***REMOVED*** + return checksumStream.getDigest(); + ***REMOVED*** catch (IOException e) { + ***REMOVED*** never happens + e.printStackTrace(); + return null; + ***REMOVED*** finally { + SVNFileUtil.closeFile(checksumStream); + ***REMOVED*** +***REMOVED*** + + public List checkMatch(String[] lines, List ls) { + Set ss = new HashSet(ls); + ss.addAll(Arrays.asList(lines)); + return new ArrayList<>(ss); +***REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java index 86b5e15..9b9e14b 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SendMail.java @@ -26,80 +26,69 @@ public class SendMail extends StandardLocalInfraAlgorithm { ***REMOVED*** TODO Auto-generated constructor stub ***REMOVED*** -public void sendNotification(String subject, String body) throws Exception { - - AnalysisLogger.getLogger().debug("Emailing System->Starting request of email in scope "+ScopeProvider.instance.get()); - + public void sendNotification(String subject, String body) throws Exception { + + AnalysisLogger.getLogger() + .debug("Emailing System->Starting request of email in scope " + ScopeProvider.instance.get()); + String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); - + if (!serviceAddress.endsWith("/")) - serviceAddress = serviceAddress+"/"; - - String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" + SecurityTokenProvider.instance.get(); - requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error + serviceAddress = serviceAddress + "/"; + + String requestForMessage = serviceAddress + "messages/writeMessageToUsers" + "?gcube-token=" + + SecurityTokenProvider.instance.get(); + requestForMessage = requestForMessage.replace("http:***REMOVED***", "https:***REMOVED***").replace(":80", ""); AnalysisLogger.getLogger().debug("Emailing System->Request url is going to be " + requestForMessage); - ***REMOVED*** put the sender, the recipients, subject and body of the mail here - subject=URLEncoder.encode(subject,"UTF-8"); - body=URLEncoder.encode(body,"UTF-8"); - String requestParameters = "&sender=dataminer&recipients="+this.getAdmins()+"&subject="+subject+"&body="+body; - - + subject = URLEncoder.encode(subject, "UTF-8"); + body = URLEncoder.encode(body, "UTF-8"); + String requestParameters = "&sender=dataminer&recipients=" + /*this.getAdmins()*/this.username(SecurityTokenProvider.instance.get()) + "&subject=" + subject + "&body=" + + body; + String response = HttpRequest.sendPostRequest(requestForMessage, requestParameters); AnalysisLogger.getLogger().debug("Emailing System->Emailing response OK "); - - if (response==null){ + + if (response == null) { Exception e = new Exception("Error in email sending response"); throw e; ***REMOVED*** ***REMOVED*** - -***REMOVED*** public void notifySubmitter(String a, String b) throws Exception { -***REMOVED*** NotificationHelper nh = new NotificationHelper(); -***REMOVED*** super.sendNotification(nh.getSubject(), -***REMOVED*** nh.getBody()); -***REMOVED*** ***REMOVED*** + ***REMOVED*** public void notifySubmitter(String a, String b) throws Exception { + ***REMOVED*** NotificationHelper nh = new NotificationHelper(); + ***REMOVED*** super.sendNotification(nh.getSubject(), + ***REMOVED*** nh.getBody()); + ***REMOVED*** ***REMOVED*** - - public String username(String token) throws ObjectNotFound, Exception{ + public String username(String token) throws ObjectNotFound, Exception { AuthorizationEntry entry = authorizationService().get(token); System.out.println(entry.getClientInfo().getId()); return entry.getClientInfo().getId(); - ***REMOVED*** - - - - - - - public String retrieveAdminRole() throws Exception{ + public String retrieveAdminRole() throws Exception { String serviceAddress = InfraRetrieval.findEmailingSystemAddress(ScopeProvider.instance.get()); - - if (!serviceAddress.endsWith("/")) - serviceAddress = serviceAddress+"/"; - - String requestForMessage = serviceAddress + "2/users/get-usernames-by-global-role"; - requestForMessage = requestForMessage.replace("https:***REMOVED***", "http:***REMOVED***").replace(":80", ""); ***REMOVED*** remove the port (or set it to 443) otherwise you get an SSL error - String requestParameters = "role-name=Administrator"+"&gcube-token=" + SecurityTokenProvider.instance.get(); - String response = HttpRequest.sendGetRequest(requestForMessage,requestParameters); + if (!serviceAddress.endsWith("/")) + serviceAddress = serviceAddress + "/"; + + String requestForMessage = serviceAddress + "2/users/get-usernames-by-global-role"; + requestForMessage = requestForMessage.replace("https:***REMOVED***", "http:***REMOVED***").replace(":80", ""); + String requestParameters = "role-name=Administrator" + "&gcube-token=" + SecurityTokenProvider.instance.get(); + + String response = HttpRequest.sendGetRequest(requestForMessage, requestParameters); System.out.println(response.toString()); - - if (response==null){ + + if (response == null) { Exception e = new Exception("Error in querying the recipient"); throw e; ***REMOVED*** return response; - - + ***REMOVED*** - - public String getAdmins() throws Exception { List s = new LinkedList(); @@ -113,67 +102,54 @@ public void sendNotification(String subject, String body) throws Exception { ***REMOVED*** s.add(this.username(SecurityTokenProvider.instance.get())); ***REMOVED*** - - System.out.println(s); - return s.toString(); + + return s.toString().replace("[", "").replace("]", ""); ***REMOVED*** - - - - + @Override public String getDescription() { ***REMOVED*** TODO Auto-generated method stub return null; ***REMOVED*** - - @Override public void init() throws Exception { ***REMOVED*** TODO Auto-generated method stub - + ***REMOVED*** - - @Override protected void process() throws Exception { ***REMOVED*** TODO Auto-generated method stub - + ***REMOVED*** - - @Override protected void setInputParameters() { ***REMOVED*** TODO Auto-generated method stub - + ***REMOVED*** - - @Override public void shutdown() { ***REMOVED*** TODO Auto-generated method stub - -***REMOVED*** - - public static void main(String[] args) throws Exception { -***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); -***REMOVED*** SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); - ScopeProvider.instance.set("/gcube/devNext"); - SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); - - - +***REMOVED*** + + public static void main(String[] args) throws Exception { + ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); + SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); + + ***REMOVED***ScopeProvider.instance.set("/gcube/devNext"); + ***REMOVED***SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); + NotificationHelper nh = new NotificationHelper(); - + SendMail sm = new SendMail(); - ***REMOVED***sm.username(SecurityTokenProvider.instance.get()); - ***REMOVED***sm.retrieveAdminRole(); - sm.getAdmins(); - ***REMOVED***sm.sendNotification("test", "test"); + sm.sendNotification(nh.getFailedSubject(), nh.getFailedBody("test failed")); + ***REMOVED*** sm.username(SecurityTokenProvider.instance.get()); + ***REMOVED*** sm.retrieveAdminRole(); + ***REMOVED***System.out.println(sm.getAdmins()); + ***REMOVED*** sm.sendNotification("test", "test"); ***REMOVED*** ***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java index 064d356..95121c7 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -19,7 +19,7 @@ public class SVNTests { SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ngalante/workspace/dataminer-pool-manager/src/main/resources/service.properties")); - Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/cnFLNHYxR1ZDa1VNdEhrTUQyQlZjaWRBVVZlUHloUitHbWJQNStIS0N6Yz0"); + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/R0ExYjFPVFBrOUlNdEhrTUQyQlZjbUJuQ20rbGFrZ0pHbWJQNStIS0N6Yz0"); ***REMOVED***algo.setClazz(algo.getClazz() + "TEST"); ***REMOVED***svnUpdater.updateSVNRProtoAlgorithmList(algo, "/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab", "Dataminer Pool Manager", "Proto");