git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@150435 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
fba1a32a23
commit
58bae45a76
|
|
@ -39,9 +39,7 @@ public class AnsibleBridge {
|
|||
private static final org.slf4j.Logger LOGGER = LoggerFactory.getLogger(AnsibleBridge.class);
|
||||
|
||||
|
||||
/**
|
||||
* The workdir for this service
|
||||
*/
|
||||
|
||||
private String dpmRoot;
|
||||
|
||||
public AnsibleBridge() {
|
||||
|
|
@ -141,10 +139,7 @@ public class AnsibleBridge {
|
|||
return worker;
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups hosts by domain and algorithm sets
|
||||
* @param clusters
|
||||
*/
|
||||
|
||||
public void printInventoryByDomainAndSets(Collection<Cluster> clusters) {
|
||||
Map<String, Set<Host>> inventory = new TreeMap<>();
|
||||
for(Cluster cluster:clusters) {
|
||||
|
|
@ -173,10 +168,7 @@ public class AnsibleBridge {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Groups hosts by algorithm sets only
|
||||
* @param clusters
|
||||
*/
|
||||
|
||||
public void printInventoryBySets(Collection<Cluster> clusters) {
|
||||
Map<String, Set<Host>> inventory = new TreeMap<>();
|
||||
for (Cluster cluster : clusters) {
|
||||
|
|
|
|||
|
|
@ -22,12 +22,7 @@ public class TemplateManager {
|
|||
return input;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the given template
|
||||
* @param templateName
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
|
||||
// private String readTemplate(String templateName) throws IOException {
|
||||
// File templateFile = new File(this.getTemplateRoot(), templateName + ".yaml");
|
||||
// System.out.println("looking for file " + templateFile.getName());
|
||||
|
|
@ -35,12 +30,7 @@ public class TemplateManager {
|
|||
// return out;
|
||||
// }
|
||||
|
||||
/**
|
||||
* Return the content of the given template
|
||||
* @param templateName
|
||||
* @return
|
||||
* @throws NoSuchElementException if no such template exists
|
||||
*/
|
||||
|
||||
// public String getTemplate(String templateName) throws NoSuchElementException {
|
||||
// String template = null;
|
||||
// try {
|
||||
|
|
|
|||
|
|
@ -7,10 +7,12 @@ import java.net.URL;
|
|||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
|
||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Host;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckPermission;
|
||||
import org.tmatesoft.svn.core.SVNException;
|
||||
import au.com.bytecode.opencsv.CSVReader;
|
||||
|
||||
|
|
@ -128,16 +130,20 @@ public class HAProxy {
|
|||
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException, SVNException {
|
||||
public static void main(String[] args) throws ObjectNotFound, Exception {
|
||||
HAProxy a = new HAProxy();
|
||||
//ScopeProvider.instance.set("/gcube/devNext/NextNext");
|
||||
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
//SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
|
||||
|
||||
//ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
|
||||
|
||||
CheckPermission test = new CheckPermission();
|
||||
CheckPermission.apply("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548", "/gcube/devNext/NextNext");
|
||||
|
||||
|
||||
//ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
// System.out.println(a.getHProxy());
|
||||
// System.out.println(a.MapCluster());
|
||||
System.out.println(a.listDataMinersByCluster("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext"));
|
||||
//System.out.println(a.listDataMinersByCluster("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext"));
|
||||
// System.out.println(a.listDataMinersByCluster());
|
||||
|
||||
// List<Dependency> list = new LinkedList<Dependency>();
|
||||
|
|
|
|||
|
|
@ -130,12 +130,7 @@ public class ISClient {
|
|||
//
|
||||
// }
|
||||
|
||||
/**
|
||||
* Return the list of hosts (dataminers) in a given VRE
|
||||
*
|
||||
* @param vreName
|
||||
* @return
|
||||
*/
|
||||
|
||||
public Collection<Host> listDataminersInVRE() {
|
||||
|
||||
boolean remote = false;
|
||||
|
|
|
|||
|
|
@ -34,16 +34,7 @@ public class AlgorithmPackageParser {
|
|||
|
||||
private static final int BUFFER_SIZE = 4096;
|
||||
|
||||
/**
|
||||
* Given an URL to an algorithm package, create an Algorithm object with its
|
||||
* metadata. Metadata are extracted from the 'info.txt' file, if any, in the
|
||||
* package.
|
||||
*
|
||||
* @param url
|
||||
* @return An Algorithm object or null if no 'info.txt' is found in the
|
||||
* package.
|
||||
* @throws IOException
|
||||
*/
|
||||
|
||||
public Algorithm parsePackage(String url) throws IOException {
|
||||
String packageMetadata = this.getPackageMetadata(url);
|
||||
if (packageMetadata == null) {
|
||||
|
|
@ -57,13 +48,7 @@ public class AlgorithmPackageParser {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the content of the metadata file from the package.
|
||||
*
|
||||
* @param url
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
|
||||
private String getPackageMetadata(String url) throws IOException {
|
||||
InputStream is = new URL(url).openStream();
|
||||
ZipInputStream zipIs = new ZipInputStream(is);
|
||||
|
|
@ -81,12 +66,7 @@ public class AlgorithmPackageParser {
|
|||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the content of a zip entry and place it in a string.
|
||||
* @param zipIn
|
||||
* @return
|
||||
* @throws IOException
|
||||
*/
|
||||
|
||||
private String getEntryContent(ZipInputStream zipIn) throws IOException {
|
||||
StringBuilder s = new StringBuilder();
|
||||
byte[] buffer = new byte[BUFFER_SIZE];
|
||||
|
|
@ -97,11 +77,7 @@ public class AlgorithmPackageParser {
|
|||
return s.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the content of the metadata file and create a key+multivalue map.
|
||||
* @param metadata
|
||||
* @return
|
||||
*/
|
||||
|
||||
private Map<String, List<String>> parseMetadata(String metadata) {
|
||||
Map<String, List<String>> out = new HashMap<String, List<String>>();
|
||||
String[] lines = metadata.split("\n");
|
||||
|
|
@ -145,11 +121,7 @@ public class AlgorithmPackageParser {
|
|||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create an Algorithm starting from its metadata
|
||||
* @param metadata
|
||||
* @return
|
||||
*/
|
||||
|
||||
// private Algorithm createAlgorithm(Map<String, List<String>> metadata) {
|
||||
// Algorithm out = new Algorithm();
|
||||
// out.setName(extractSingleValue(metadata, METADATA_ALGORITHM_NAME));
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.rest;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URL;
|
||||
|
|
@ -12,6 +14,10 @@ import javax.ws.rs.Path;
|
|||
import javax.ws.rs.Produces;
|
||||
import javax.ws.rs.QueryParam;
|
||||
|
||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
|
||||
|
|
@ -26,7 +32,6 @@ public class RestPoolManager implements PoolManager {
|
|||
|
||||
private DataminerPoolManager service = new DataminerPoolManager();
|
||||
|
||||
//TODO: for all methods accept also a callback url to send the log.
|
||||
|
||||
@GET
|
||||
@Path("/algorithm/stage")
|
||||
|
|
@ -72,14 +77,23 @@ public class RestPoolManager implements PoolManager {
|
|||
return null;
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws IOException, InterruptedException, SVNException {
|
||||
// // System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
|
||||
public static void main(String[] args) throws ObjectNotFound, Exception {
|
||||
|
||||
|
||||
|
||||
// // System.out.println(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
|
||||
// // ProxySelector.setDefault(new
|
||||
// // PropertiesBasedProxySelector("/home/ngalante/.proxy-settings"));
|
||||
//
|
||||
// ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
// //SecurityTokenProvider.instance.set("3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462");
|
||||
// RestPoolManager a = new RestPoolManager();
|
||||
SecurityTokenProvider.instance.set("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
|
||||
|
||||
AuthorizationEntry entry = authorizationService().get("708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
|
||||
System.out.println(entry.getContext());
|
||||
|
||||
|
||||
//RestPoolManager a = new RestPoolManager();
|
||||
|
||||
// a.stageAlgorithm("http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", category);
|
||||
// //a.publishAlgorithm("http://data.d4science.org/MnovRjZIdGV5WlB0WXE5NVNaZnRoRVg0SU8xZWpWQlFHbWJQNStIS0N6Yz0", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548","/gcube/devNext/NextNext");
|
||||
// // PoolManager aa = new DataminerPoolManager();
|
||||
|
|
@ -94,7 +108,6 @@ public class RestPoolManager implements PoolManager {
|
|||
stageAlgorithm(Rproto caller token,pacchetto, category)
|
||||
http://node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/stage?gcube-token=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL
|
||||
|
||||
|
||||
publishAlgorithm(Rproto caller token, pacchetto, category, target token, target prod vre)
|
||||
node2-d-d4s.d4science.org:8080/dataminer-pool-manager-1.0.0-SNAPSHOT/rest/algorithm/add?gcube-token=708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548&algorithmPackageURL=http://data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0&category=ICHTHYOP_MODEL&targetVREToken=3a23bfa4-4dfe-44fc-988f-194b91071dd2-843339462&targetVRE=/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab
|
||||
|
||||
|
|
|
|||
|
|
@ -28,7 +28,6 @@ public class DataminerPoolManager {
|
|||
}
|
||||
|
||||
|
||||
|
||||
public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
|
||||
|
||||
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
|
||||
|
|
@ -43,7 +42,7 @@ public class DataminerPoolManager {
|
|||
|
||||
Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
|
||||
|
||||
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE);
|
||||
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, targetVREToken);
|
||||
String id = job.start();
|
||||
return id;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckPermission;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
|
||||
public class ProductionPublishingJob extends DMPMJob {
|
||||
|
|
@ -9,21 +12,28 @@ public class ProductionPublishingJob extends DMPMJob {
|
|||
private Algorithm algorithm;
|
||||
private Cluster prodCluster;
|
||||
private String targetVREName;
|
||||
private String targetVREToken;
|
||||
|
||||
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName) {
|
||||
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName, String targetVREToken) {
|
||||
super(svnUpdater);
|
||||
this.algorithm = algorithm;
|
||||
this.prodCluster = prodCluster;
|
||||
this.targetVREName = targetVREName;
|
||||
this.targetVREToken = targetVREToken;
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
protected void execute() {
|
||||
try {
|
||||
this.svnUpdater.updateProdDeps(this.algorithm);
|
||||
|
||||
if (CheckPermission.apply(targetVREToken,targetVREName)){
|
||||
|
||||
this.svnUpdater.updateProdDeps(this.algorithm);
|
||||
|
||||
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
|
||||
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
|
||||
|
||||
}
|
||||
|
||||
// int ret = this.executeAnsibleWorker(
|
||||
// createWorker(this.algorithm, this.prodCluster, false, "gcube"));
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.service;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.util;
|
||||
|
||||
import org.gcube.common.authorization.client.exceptions.ObjectNotFound;
|
||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
|
||||
import static org.gcube.common.authorization.client.Constants.authorizationService;
|
||||
|
||||
|
||||
public class CheckPermission {
|
||||
|
||||
public CheckPermission() {
|
||||
}
|
||||
|
||||
public static boolean apply(String VREToken, String vre) throws ObjectNotFound, Exception {
|
||||
|
||||
AuthorizationEntry entry = authorizationService().get(VREToken);
|
||||
if (entry.getContext().equals(vre)) {
|
||||
System.out.println("OK!");
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
#YML node file
|
||||
#STAGING_HOST: dataminer-proto-ghost.d4science.org
|
||||
STAGING_HOST: dataminer1-devnext.d4science.org
|
||||
STAGING_HOST: dataminer-proto-ghost.d4science.org
|
||||
#STAGING_HOST: dataminer1-devnext.d4science.org
|
||||
SVN_REPO: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
|
||||
#HAPROXY_CSV: http://data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue