From 994f1abad54cc4b531f7f7e08b567ce2053658a9 Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Mon, 19 Jun 2017 16:34:44 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@150435 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../ansiblebridge/AnsibleBridge.java | 14 ++----- .../template/TemplateManager.java | 14 +------ .../poolmanager/clients/ISClient.java | 7 +--- .../process/AlgorithmPackageParser.java | 38 +++---------------- .../poolmanager/rest/RestPoolManager.java | 25 +++++++++--- .../service/DataminerPoolManager.java | 3 +- .../service/ProductionPublishingJob.java | 18 +++++++-- .../poolmanager/service/StagingJob.java | 4 ++ .../poolmanager/util/CheckPermission.java | 27 +++++++++++++ src/main/resources/service.properties | 4 +- 10 files changed, 78 insertions(+), 76 deletions(-) create mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index a35156f..403ea41 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -39,9 +39,7 @@ public class AnsibleBridge { private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class); - /** - * The workdir for this service - */ + private String dpmRoot; public AnsibleBridge() { @@ -141,10 +139,7 @@ public class AnsibleBridge { return worker; ***REMOVED*** - /** - * Groups hosts by domain and algorithm sets - * @param clusters - */ + public void printInventoryByDomainAndSets(Collection clusters) { Map> inventory = new TreeMap<>(); for(Cluster cluster:clusters) { @@ -173,10 +168,7 @@ public class AnsibleBridge { ***REMOVED*** ***REMOVED*** - /** - * Groups hosts by algorithm sets only - * @param clusters - */ + public void printInventoryBySets(Collection clusters) { Map> inventory = new TreeMap<>(); for (Cluster cluster : clusters) { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java index bddb914..257c01b 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java @@ -22,12 +22,7 @@ public class TemplateManager { return input; ***REMOVED*** - /** - * Read the given template - * @param templateName - * @return - * @throws IOException - */ + ***REMOVED*** private String readTemplate(String templateName) throws IOException { ***REMOVED*** File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml"); ***REMOVED*** System.out.println("looking for file " + templateFile.getName()); @@ -35,12 +30,7 @@ public class TemplateManager { ***REMOVED*** return out; ***REMOVED*** ***REMOVED*** - /** - * Return the content of the given template - * @param templateName - * @return - * @throws NoSuchElementException if no such template exists - */ + ***REMOVED*** public String getTemplate(String templateName) throws NoSuchElementException { ***REMOVED*** String template = null; ***REMOVED*** try { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java index c1c0ce8..75055bf 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java @@ -130,12 +130,7 @@ public class ISClient { ***REMOVED*** ***REMOVED*** ***REMOVED*** - /** - * Return the list of hosts (dataminers) in a given VRE - * - * @param vreName - * @return - */ + public Collection listDataminersInVRE() { boolean remote = false; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java index 984c440..1af3d7c 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java @@ -34,16 +34,7 @@ public class AlgorithmPackageParser { private static final int BUFFER_SIZE = 4096; - /** - * Given an URL to an algorithm package, create an Algorithm object with its - * metadata. Metadata are extracted from the 'info.txt' file, if any, in the - * package. - * - * @param url - * @return An Algorithm object or null if no 'info.txt' is found in the - * package. - * @throws IOException - */ + public Algorithm parsePackage(String url) throws IOException { String packageMetadata = this.getPackageMetadata(url); if (packageMetadata == null) { @@ -57,13 +48,7 @@ public class AlgorithmPackageParser { ***REMOVED*** ***REMOVED*** - /** - * Extract the content of the metadata file from the package. - * - * @param url - * @return - * @throws IOException - */ + private String getPackageMetadata(String url) throws IOException { InputStream is = new URL(url).openStream(); ZipInputStream zipIs = new ZipInputStream(is); @@ -81,12 +66,7 @@ public class AlgorithmPackageParser { return out; ***REMOVED*** - /** - * Read the content of a zip entry and place it in a string. - * @param zipIn - * @return - * @throws IOException - */ + private String getEntryContent(ZipInputStream zipIn) throws IOException { StringBuilder s = new StringBuilder(); byte[] buffer = new byte[BUFFER_SIZE]; @@ -97,11 +77,7 @@ public class AlgorithmPackageParser { return s.toString(); ***REMOVED*** - /** - * Parse the content of the metadata file and create a key+multivalue map. - * @param metadata - * @return - */ + private Map> parseMetadata(String metadata) { Map> out = new HashMap>(); String[] lines = metadata.split("\n"); @@ -145,11 +121,7 @@ public class AlgorithmPackageParser { return out; ***REMOVED*** - /** - * Create an Algorithm starting from its metadata - * @param metadata - * @return - */ + ***REMOVED*** private Algorithm createAlgorithm(Map> metadata) { ***REMOVED*** Algorithm out = new Algorithm(); ***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME)); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java index 86b56f2..4e45f4d 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java @@ -1,5 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.rest; +import static org.gcube.common.authorization.client.Constants.authorizationService; + ***REMOVED*** import java.net.MalformedURLException; ***REMOVED*** @@ -12,6 +14,10 @@ import javax.ws.rs.Path; import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; +***REMOVED*** +import org.gcube.common.authorization.library.AuthorizationEntry; +***REMOVED*** +***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager; import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; @@ -26,7 +32,6 @@ public class RestPoolManager implements PoolManager { private DataminerPoolManager service = new DataminerPoolManager(); - ***REMOVED***TODO: for all methods accept also a callback url to send the log. @GET @Path("/algorithm/stage") @@ -72,14 +77,23 @@ public class RestPoolManager implements PoolManager { return null; ***REMOVED*** - public static void main(String[] args) throws IOException, InterruptedException, SVNException { -***REMOVED*** ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); +***REMOVED*** + + + + ***REMOVED*** ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); ***REMOVED*** ***REMOVED*** ProxySelector.setDefault(new ***REMOVED*** ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings")); ***REMOVED*** ***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); -***REMOVED*** ***REMOVED***SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462"); -***REMOVED*** RestPoolManager a = new RestPoolManager(); + SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); + + AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); + System.out.println(entry.getContext()); + + + ***REMOVED***RestPoolManager a = new RestPoolManager(); + ***REMOVED*** a.stageAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", category); ***REMOVED*** ***REMOVED***a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext"); ***REMOVED*** ***REMOVED*** PoolManager aa = new DataminerPoolManager(); @@ -94,7 +108,6 @@ public class RestPoolManager implements PoolManager { stageAlgorithm(Rproto caller token,pacchetto, category) http:***REMOVED***node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&algorithmPackageURL=http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL - publishAlgorithm(Rproto caller token, pacchetto, category, target token, target prod vre) node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/add?gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548&algorithmPackageURL=http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL&targetVREToken=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&targetVRE=/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 6a50392..bca8df2 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -28,7 +28,6 @@ public class DataminerPoolManager { ***REMOVED*** - public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException { Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); @@ -43,7 +42,7 @@ public class DataminerPoolManager { Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); - DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE); + DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, targetVREToken); String id = job.start(); return id; ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java index 01bfbea..ac1fd34 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -1,7 +1,10 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; +***REMOVED*** +***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; ***REMOVED*** +***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; public class ProductionPublishingJob extends DMPMJob { @@ -9,21 +12,28 @@ public class ProductionPublishingJob extends DMPMJob { private Algorithm algorithm; private Cluster prodCluster; private String targetVREName; + private String targetVREToken; - public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName) { + public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName, String targetVREToken) { super(svnUpdater); this.algorithm = algorithm; this.prodCluster = prodCluster; this.targetVREName = targetVREName; + this.targetVREToken = targetVREToken; ***REMOVED*** - + @Override protected void execute() { try { - this.svnUpdater.updateProdDeps(this.algorithm); + + if (CheckPermission.apply(targetVREToken,targetVREName)){ + + this.svnUpdater.updateProdDeps(this.algorithm); - this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod"); + this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod"); + ***REMOVED*** + ***REMOVED*** int ret = this.executeAnsibleWorker( ***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube")); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index 0b8940e..c6df4c5 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -1,5 +1,9 @@ package org.gcube.dataanalysis.dataminer.poolmanager.service; +import static org.gcube.common.authorization.client.Constants.authorizationService; + +import org.gcube.common.authorization.library.AuthorizationEntry; +***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java new file mode 100644 index 0000000..db8b151 --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java @@ -0,0 +1,27 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +***REMOVED*** +import org.gcube.common.authorization.library.AuthorizationEntry; +***REMOVED*** +***REMOVED*** + +import static org.gcube.common.authorization.client.Constants.authorizationService; + + +public class CheckPermission { + + public CheckPermission() { +***REMOVED*** + + public static boolean apply(String VREToken, String vre) throws ObjectNotFound, Exception { + + AuthorizationEntry entry = authorizationService().get(VREToken); + if (entry.getContext().equals(vre)) { + System.out.println("OK!"); + return true; + ***REMOVED*** + + return false; + +***REMOVED*** +***REMOVED*** diff --git a/src/main/resources/service.properties b/src/main/resources/service.properties index 498c796..d4f49f2 100644 --- a/src/main/resources/service.properties +++ b/src/main/resources/service.properties @@ -1,6 +1,6 @@ #YML node file -#STAGING_HOST: dataminer-proto-ghost.d4science.org -STAGING_HOST: dataminer1-devnext.d4science.org +STAGING_HOST: dataminer-proto-ghost.d4science.org +#STAGING_HOST: dataminer1-devnext.d4science.org SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ #HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0