gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@150435 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-06-19 16:34:44 +00:00
parent cd23218da4
commit 994f1abad5
10 changed files with 78 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -39,9 +39,7 @@ public class AnsibleBridge {
private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class);
/**
* The workdir for this service
*/
private String dpmRoot;
public AnsibleBridge() {
@ -141,10 +139,7 @@ public class AnsibleBridge {
return worker;
***REMOVED***
/**
* Groups hosts by domain and algorithm sets
* @param clusters
*/
public void printInventoryByDomainAndSets(Collection<Cluster> clusters) {
Map<String, Set<Host>> inventory = new TreeMap<>();
for(Cluster cluster:clusters) {
@ -173,10 +168,7 @@ public class AnsibleBridge {
***REMOVED***
***REMOVED***
/**
* Groups hosts by algorithm sets only
* @param clusters
*/
public void printInventoryBySets(Collection<Cluster> clusters) {
Map<String, Set<Host>> inventory = new TreeMap<>();
for (Cluster cluster : clusters) {

14
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java
View File

@ -22,12 +22,7 @@ public class TemplateManager {
return input;
***REMOVED***
/**
* Read the given template
* @param templateName
* @return
* @throws IOException
*/
***REMOVED*** private String readTemplate(String templateName) throws IOException {
***REMOVED*** File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml");
***REMOVED*** System.out.println("looking for file " + templateFile.getName());
@ -35,12 +30,7 @@ public class TemplateManager {
***REMOVED*** return out;
***REMOVED*** ***REMOVED***
/**
* Return the content of the given template
* @param templateName
* @return
* @throws NoSuchElementException if no such template exists
*/
***REMOVED*** public String getTemplate(String templateName) throws NoSuchElementException {
***REMOVED*** String template = null;
***REMOVED*** try {

7
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/ISClient.java
View File

@ -130,12 +130,7 @@ public class ISClient {
***REMOVED***
***REMOVED*** ***REMOVED***
/**
* Return the list of hosts (dataminers) in a given VRE
*
* @param vreName
* @return
*/
public Collection<Host> listDataminersInVRE() {
boolean remote = false;

38
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/process/AlgorithmPackageParser.java
View File

@ -34,16 +34,7 @@ public class AlgorithmPackageParser {
private static final int BUFFER_SIZE = 4096;
/**
* Given an URL to an algorithm package, create an Algorithm object with its
* metadata. Metadata are extracted from the 'info.txt' file, if any, in the
* package.
*
* @param url
* @return An Algorithm object or null if no 'info.txt' is found in the
* package.
* @throws IOException
*/
public Algorithm parsePackage(String url) throws IOException {
String packageMetadata = this.getPackageMetadata(url);
if (packageMetadata == null) {
@ -57,13 +48,7 @@ public class AlgorithmPackageParser {
***REMOVED***
***REMOVED***
/**
* Extract the content of the metadata file from the package.
*
* @param url
* @return
* @throws IOException
*/
private String getPackageMetadata(String url) throws IOException {
InputStream is = new URL(url).openStream();
ZipInputStream zipIs = new ZipInputStream(is);
@ -81,12 +66,7 @@ public class AlgorithmPackageParser {
return out;
***REMOVED***
/**
* Read the content of a zip entry and place it in a string.
* @param zipIn
* @return
* @throws IOException
*/
private String getEntryContent(ZipInputStream zipIn) throws IOException {
StringBuilder s = new StringBuilder();
byte[] buffer = new byte[BUFFER_SIZE];
@ -97,11 +77,7 @@ public class AlgorithmPackageParser {
return s.toString();
***REMOVED***
/**
* Parse the content of the metadata file and create a key+multivalue map.
* @param metadata
* @return
*/
private Map<String, List<String>> parseMetadata(String metadata) {
Map<String, List<String>> out = new HashMap<String, List<String>>();
String[] lines = metadata.split("\n");
@ -145,11 +121,7 @@ public class AlgorithmPackageParser {
return out;
***REMOVED***
/**
* Create an Algorithm starting from its metadata
* @param metadata
* @return
*/
***REMOVED*** private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
***REMOVED*** Algorithm out = new Algorithm();
***REMOVED*** out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));

25
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/rest/RestPoolManager.java
View File

@ -1,5 +1,7 @@
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
import static org.gcube.common.authorization.client.Constants.authorizationService;
***REMOVED***
import java.net.MalformedURLException;
***REMOVED***
@ -12,6 +14,10 @@ import javax.ws.rs.Path;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
***REMOVED***
import org.gcube.common.authorization.library.AuthorizationEntry;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
@ -26,7 +32,6 @@ public class RestPoolManager implements PoolManager {
private DataminerPoolManager service = new DataminerPoolManager();
***REMOVED***TODO: for all methods accept also a callback url to send the log.
@GET
@Path("/algorithm/stage")
@ -72,14 +77,23 @@ public class RestPoolManager implements PoolManager {
return null;
***REMOVED***
public static void main(String[] args) throws IOException, InterruptedException, SVNException {
***REMOVED*** ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
***REMOVED***
***REMOVED*** ***REMOVED*** System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
***REMOVED*** ***REMOVED*** ProxySelector.setDefault(new
***REMOVED*** ***REMOVED*** PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
***REMOVED***
***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
***REMOVED*** ***REMOVED***SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
***REMOVED*** RestPoolManager a = new RestPoolManager();
SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
System.out.println(entry.getContext());
***REMOVED***RestPoolManager a = new RestPoolManager();
***REMOVED*** a.stageAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", category);
***REMOVED*** ***REMOVED***a.publishAlgorithm("http:***REMOVED***data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext");
***REMOVED*** ***REMOVED*** PoolManager aa = new DataminerPoolManager();
@ -94,7 +108,6 @@ public class RestPoolManager implements PoolManager {
stageAlgorithm(Rproto caller token,pacchetto, category)
http:***REMOVED***node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&algorithmPackageURL=http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL
publishAlgorithm(Rproto caller token, pacchetto, category, target token, target prod vre)
node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/add?gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548&algorithmPackageURL=http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL&targetVREToken=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&targetVRE=/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab

3
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -28,7 +28,6 @@ public class DataminerPoolManager {
***REMOVED***
public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
@ -43,7 +42,7 @@ public class DataminerPoolManager {
Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE);
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, targetVREToken);
String id = job.start();
return id;
***REMOVED***

18
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
View File

@ -1,7 +1,10 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
public class ProductionPublishingJob extends DMPMJob {
@ -9,21 +12,28 @@ public class ProductionPublishingJob extends DMPMJob {
private Algorithm algorithm;
private Cluster prodCluster;
private String targetVREName;
private String targetVREToken;
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName) {
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName, String targetVREToken) {
super(svnUpdater);
this.algorithm = algorithm;
this.prodCluster = prodCluster;
this.targetVREName = targetVREName;
this.targetVREToken = targetVREToken;
***REMOVED***
@Override
protected void execute() {
try {
this.svnUpdater.updateProdDeps(this.algorithm);
if (CheckPermission.apply(targetVREToken,targetVREName)){
this.svnUpdater.updateProdDeps(this.algorithm);
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
***REMOVED***
***REMOVED*** int ret = this.executeAnsibleWorker(
***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube"));

4
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
View File

@ -1,5 +1,9 @@
package org.gcube.dataanalysis.dataminer.poolmanager.service;
import static org.gcube.common.authorization.client.Constants.authorizationService;
import org.gcube.common.authorization.library.AuthorizationEntry;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;

27
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckPermission.java Normal file
View File

@ -0,0 +1,27 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
***REMOVED***
import org.gcube.common.authorization.library.AuthorizationEntry;
***REMOVED***
***REMOVED***
import static org.gcube.common.authorization.client.Constants.authorizationService;
public class CheckPermission {
public CheckPermission() {
***REMOVED***
public static boolean apply(String VREToken, String vre) throws ObjectNotFound, Exception {
AuthorizationEntry entry = authorizationService().get(VREToken);
if (entry.getContext().equals(vre)) {
System.out.println("OK!");
return true;
***REMOVED***
return false;
***REMOVED***
***REMOVED***

4
src/main/resources/service.properties
View File

@ -1,6 +1,6 @@
#YML node file
#STAGING_HOST: dataminer-proto-ghost.d4science.org
STAGING_HOST: dataminer1-devnext.d4science.org
STAGING_HOST: dataminer-proto-ghost.d4science.org
#STAGING_HOST: dataminer1-devnext.d4science.org
SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
#HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0