git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@151353 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
bb2c5af9ee
commit
60e22e39f6
|
@ -289,4 +289,31 @@ public class AlgorithmPackageParser {
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
public static void main(String[] args) {
|
||||||
|
AlgorithmPackageParser ap = new AlgorithmPackageParser();
|
||||||
|
String txt =
|
||||||
|
"Username: giancarlo.panichi\n"+
|
||||||
|
"Full Name: Giancarlo Panichi\n"+
|
||||||
|
"Email: g.panichi@isti.cnr.it\n"+
|
||||||
|
"Language: R\n"+
|
||||||
|
"Algorithm Name: RBLACKBOX\n"+
|
||||||
|
"Class Name: org.gcube.dataanalysis.executor.rscripts.RBlackBox\n"+
|
||||||
|
"Algorithm Description: RBlackBox\n"+
|
||||||
|
"Algorithm Category: BLACK_BOX\n"+
|
||||||
|
"Interpreter Version: 3.2.1\n"+
|
||||||
|
"Packages:\n"+
|
||||||
|
"Package Name: DBI\n"+
|
||||||
|
"Package Name: RPostgreSQL\n"+
|
||||||
|
"Package Name: raster\n"+
|
||||||
|
"Package Name: maptools\n"+
|
||||||
|
"Package Name: sqldf\n"+
|
||||||
|
"Package Name: RJSONIO\n"+
|
||||||
|
"Package Name: httr \n"+
|
||||||
|
"Package Name: data.table";
|
||||||
|
|
||||||
|
|
||||||
|
ap.parseMetadata(txt);
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
|
@ -9,10 +9,12 @@ import java.net.UnknownHostException;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
import java.util.Set;
|
import java.util.Set;
|
||||||
|
|
||||||
|
import javax.servlet.ServletContext;
|
||||||
import javax.ws.rs.GET;
|
import javax.ws.rs.GET;
|
||||||
import javax.ws.rs.Path;
|
import javax.ws.rs.Path;
|
||||||
import javax.ws.rs.Produces;
|
import javax.ws.rs.Produces;
|
||||||
import javax.ws.rs.QueryParam;
|
import javax.ws.rs.QueryParam;
|
||||||
|
import javax.ws.rs.core.Context;
|
||||||
|
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
import org.gcube.common.authorization.library.AuthorizationEntry;
|
import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||||
|
@ -21,6 +23,8 @@ import org.gcube.common.authorization.library.AuthorizationEntry;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
|
import org.gcube.dataanalysis.dataminer.poolmanager.service.DataminerPoolManager;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
|
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
|
||||||
|
import org.gcube.smartgears.ContextProvider;
|
||||||
|
import org.gcube.smartgears.context.application.ApplicationContext;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
import org.tmatesoft.svn.core.SVNException;
|
import org.tmatesoft.svn.core.SVNException;
|
||||||
|
@ -28,10 +32,18 @@ import org.tmatesoft.svn.core.SVNException;
|
||||||
@Path("/")
|
@Path("/")
|
||||||
public class RestPoolManager implements PoolManager {
|
public class RestPoolManager implements PoolManager {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
***REMOVED***@Context ServletContext context;
|
||||||
|
|
||||||
private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class);
|
private static final Logger LOGGER = LoggerFactory.getLogger(RestPoolManager.class);
|
||||||
|
|
||||||
private DataminerPoolManager service = new DataminerPoolManager();
|
private DataminerPoolManager service = new DataminerPoolManager();
|
||||||
|
|
||||||
|
***REMOVED***@Context
|
||||||
|
private ApplicationContext context = ContextProvider.get();
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@GET
|
@GET
|
||||||
@Path("/algorithm/stage")
|
@Path("/algorithm/stage")
|
||||||
|
@ -40,7 +52,8 @@ public class RestPoolManager implements PoolManager {
|
||||||
@QueryParam("algorithmPackageURL") String algorithmPackageURL
|
@QueryParam("algorithmPackageURL") String algorithmPackageURL
|
||||||
/*@QueryParam("category") String category*/) throws IOException, InterruptedException {
|
/*@QueryParam("category") String category*/) throws IOException, InterruptedException {
|
||||||
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
|
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
|
||||||
return this.service.stageAlgorithm(algo);
|
String env = context.application().getInitParameter("Environment");
|
||||||
|
return this.service.stageAlgorithm(algo,env);
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
@ -53,7 +66,8 @@ public class RestPoolManager implements PoolManager {
|
||||||
@QueryParam("targetVRE") String targetVRE
|
@QueryParam("targetVRE") String targetVRE
|
||||||
/*@QueryParam("category") String category*/) throws IOException, InterruptedException {
|
/*@QueryParam("category") String category*/) throws IOException, InterruptedException {
|
||||||
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
|
Algorithm algo = AlgorithmBuilder.create(algorithmPackageURL);
|
||||||
return this.service.publishAlgorithm(algo, targetVREToken, targetVRE);
|
String env = context.application().getInitParameter("Environment");
|
||||||
|
return this.service.publishAlgorithm(algo, targetVREToken, targetVRE,env);
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
/*
|
/*
|
||||||
|
|
|
@ -28,21 +28,23 @@ public class DataminerPoolManager {
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
public String stageAlgorithm(Algorithm algo) throws IOException, InterruptedException {
|
public String stageAlgorithm(Algorithm algo, String env) throws IOException, InterruptedException {
|
||||||
|
|
||||||
|
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(env);
|
||||||
|
|
||||||
|
|
||||||
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
|
|
||||||
***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
|
***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
|
||||||
|
|
||||||
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ ScopeProvider.instance.get());
|
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ ScopeProvider.instance.get(), env);
|
||||||
String id = job.start();
|
String id = job.start();
|
||||||
return id;
|
return id;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
public String publishAlgorithm(Algorithm algo, String targetVREToken, String targetVRE) throws IOException, InterruptedException {
|
public String publishAlgorithm(Algorithm algo, String targetVREToken, String targetVRE, String env) throws IOException, InterruptedException {
|
||||||
|
|
||||||
***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
|
***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
|
||||||
|
|
||||||
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, /*prodCluster,*/ targetVRE, targetVREToken);
|
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, /*prodCluster,*/ targetVRE, targetVREToken,env);
|
||||||
String id = job.start();
|
String id = job.start();
|
||||||
return id;
|
return id;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
|
@ -20,13 +20,16 @@ public class ProductionPublishingJob extends DMPMJob {
|
||||||
***REMOVED***private Cluster prodCluster;
|
***REMOVED***private Cluster prodCluster;
|
||||||
private String targetVREName;
|
private String targetVREName;
|
||||||
private String targetVREToken;
|
private String targetVREToken;
|
||||||
|
private String env;
|
||||||
|
|
||||||
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, /*Cluster prodCluster,*/ String targetVREName, String targetVREToken) throws FileNotFoundException, UnsupportedEncodingException {
|
|
||||||
|
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, /*Cluster prodCluster,*/ String targetVREName, String targetVREToken, String env) throws FileNotFoundException, UnsupportedEncodingException {
|
||||||
super(svnUpdater);
|
super(svnUpdater);
|
||||||
this.algorithm = algorithm;
|
this.algorithm = algorithm;
|
||||||
***REMOVED***this.prodCluster = prodCluster;
|
***REMOVED***this.prodCluster = prodCluster;
|
||||||
this.targetVREName = targetVREName;
|
this.targetVREName = targetVREName;
|
||||||
this.targetVREToken = targetVREToken;
|
this.targetVREToken = targetVREToken;
|
||||||
|
this.env= env;
|
||||||
|
|
||||||
|
|
||||||
this.jobLogs = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
|
this.jobLogs = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
|
||||||
|
@ -42,7 +45,7 @@ public class ProductionPublishingJob extends DMPMJob {
|
||||||
if (CheckPermission.apply(targetVREToken,targetVREName)){
|
if (CheckPermission.apply(targetVREToken,targetVREName)){
|
||||||
|
|
||||||
***REMOVED***this.svnUpdater.updateProdDeps(this.algorithm);
|
***REMOVED***this.svnUpdater.updateProdDeps(this.algorithm);
|
||||||
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, this.algorithm.getFullname(), "Prod");
|
this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.targetVREName, this.algorithm.getFullname(), env);
|
||||||
this.getStatus(9);
|
this.getStatus(9);
|
||||||
sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBodyRelease(this.buildInfo()));
|
sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBodyRelease(this.buildInfo()));
|
||||||
return;
|
return;
|
||||||
|
|
|
@ -27,10 +27,12 @@ public class StagingJob extends DMPMJob {
|
||||||
private Cluster stagingCluster;
|
private Cluster stagingCluster;
|
||||||
***REMOVED*** private Cluster rProtoCluster;
|
***REMOVED*** private Cluster rProtoCluster;
|
||||||
private String rProtoVREName;
|
private String rProtoVREName;
|
||||||
|
private String env;
|
||||||
|
|
||||||
|
|
||||||
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm,
|
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm,
|
||||||
Cluster stagingCluster, /* Cluster rProtoCluster, */
|
Cluster stagingCluster, /* Cluster rProtoCluster, */
|
||||||
String rProtoVREName) throws FileNotFoundException, UnsupportedEncodingException {
|
String rProtoVREName, String env) throws FileNotFoundException, UnsupportedEncodingException {
|
||||||
super(svnUpdater);
|
super(svnUpdater);
|
||||||
this.jobLogs = new File(
|
this.jobLogs = new File(
|
||||||
System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
|
System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
|
||||||
|
@ -40,6 +42,8 @@ public class StagingJob extends DMPMJob {
|
||||||
this.stagingCluster = stagingCluster;
|
this.stagingCluster = stagingCluster;
|
||||||
***REMOVED*** this.rProtoCluster = rProtoCluster;
|
***REMOVED*** this.rProtoCluster = rProtoCluster;
|
||||||
this.rProtoVREName = rProtoVREName;
|
this.rProtoVREName = rProtoVREName;
|
||||||
|
this.env = env;
|
||||||
|
|
||||||
|
|
||||||
***REMOVED***File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
|
***REMOVED***File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
|
||||||
***REMOVED***PrintWriter writer = new PrintWriter(m, "UTF-8");
|
***REMOVED***PrintWriter writer = new PrintWriter(m, "UTF-8");
|
||||||
|
@ -58,17 +62,18 @@ public class StagingJob extends DMPMJob {
|
||||||
try {
|
try {
|
||||||
|
|
||||||
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
|
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
|
||||||
this.svnUpdater.getRProtoDependencyFile(this.algorithm.getLanguage()),
|
this.svnUpdater.getDependencyFile(this.algorithm.getLanguage(),env),
|
||||||
this.algorithm.getDependencies());
|
this.algorithm.getDependencies());
|
||||||
|
|
||||||
if (!undefinedDependencies.isEmpty()) {
|
if (!undefinedDependencies.isEmpty()) {
|
||||||
|
|
||||||
String message = "Following dependencies are not defined:\n";
|
String message = "Following dependencies are not defined:\n";
|
||||||
for (String n : undefinedDependencies) {
|
for (String n : undefinedDependencies) {
|
||||||
message += "\n" + n;
|
message += "\n" + n +"\n";
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
this.getStatus(2);
|
||||||
|
|
||||||
sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(this.buildInfo()+message));
|
sm.sendNotification(nh.getFailedSubject() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+this.buildInfo()));
|
||||||
return;
|
return;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
@ -84,14 +89,14 @@ public class StagingJob extends DMPMJob {
|
||||||
if (ret == 0) {
|
if (ret == 0) {
|
||||||
this.getStatus(0);
|
this.getStatus(0);
|
||||||
|
|
||||||
if (b.checkMethod(a.getStagingHost(), SecurityTokenProvider.instance.get())
|
if (b.checkMethod(a.getHost(this.env), SecurityTokenProvider.instance.get())
|
||||||
&& (b.algoExists(this.algorithm))) {
|
&& (b.algoExists(this.algorithm, this.env))) {
|
||||||
|
|
||||||
System.out.println("Interface check ok!");
|
System.out.println("Interface check ok!");
|
||||||
System.out.println("Both the files exist at the correct path!");
|
System.out.println("Both the files exist at the correct path!");
|
||||||
|
|
||||||
this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName,
|
this.svnUpdater.updateSVNAlgorithmList(this.algorithm, this.rProtoVREName,
|
||||||
this.algorithm.getFullname(), "Proto");
|
this.algorithm.getFullname(), env);
|
||||||
|
|
||||||
this.getStatus(9);
|
this.getStatus(9);
|
||||||
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody(this.buildInfo()));
|
sm.sendNotification(nh.getSuccessSubject() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody(this.buildInfo()));
|
||||||
|
@ -148,7 +153,7 @@ public class StagingJob extends DMPMJob {
|
||||||
"Algorithm details:\n"+"\n"+
|
"Algorithm details:\n"+"\n"+
|
||||||
"User: "+this.algorithm.getFullname()+"\n"+
|
"User: "+this.algorithm.getFullname()+"\n"+
|
||||||
"Algorithm name: "+this.algorithm.getName()+"\n"+
|
"Algorithm name: "+this.algorithm.getName()+"\n"+
|
||||||
"Staging DataMiner Host: "+ a.getStagingHost()+"\n"+
|
"Staging DataMiner Host: "+ a.getHost(this.env)+"\n"+
|
||||||
"Caller VRE: "+rProtoVREName+"\n"+
|
"Caller VRE: "+rProtoVREName+"\n"+
|
||||||
"Target VRE: "+rProtoVREName+"\n";
|
"Target VRE: "+rProtoVREName+"\n";
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
|
@ -85,7 +85,7 @@ public class CheckMethod {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public boolean algoExists(Algorithm a) throws Exception{
|
public boolean algoExists(Algorithm a, String env) throws Exception{
|
||||||
|
|
||||||
|
|
||||||
File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar");
|
File file = new File("/home/gcube/wps_algorithms/algorithms/"+a.getName()+".jar");
|
||||||
|
@ -96,9 +96,9 @@ public class CheckMethod {
|
||||||
System.out.println("Second file is located to: "+file2.getPath());
|
System.out.println("Second file is located to: "+file2.getPath());
|
||||||
|
|
||||||
|
|
||||||
if ((this.doesExist(file.getPath())) && (this.doesExist(file2.getPath()))){
|
if ((this.doesExist(file.getPath(),env)) && (this.doesExist(file2.getPath(),env))){
|
||||||
this.copyFromDmToSVN(file);
|
this.copyFromDmToSVN(file,env);
|
||||||
this.copyFromDmToSVN(file2);
|
this.copyFromDmToSVN(file2,env);
|
||||||
|
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
|
@ -109,7 +109,7 @@ public class CheckMethod {
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
public boolean doesExist(String path) throws Exception {
|
public boolean doesExist(String path, String env) throws Exception {
|
||||||
JSch jsch = new JSch();
|
JSch jsch = new JSch();
|
||||||
Session session = null;
|
Session session = null;
|
||||||
Channel channel = null;
|
Channel channel = null;
|
||||||
|
@ -124,7 +124,7 @@ public class CheckMethod {
|
||||||
jsch.addIdentity(privateKey);
|
jsch.addIdentity(privateKey);
|
||||||
System.out.println("Private Key Added.");
|
System.out.println("Private Key Added.");
|
||||||
|
|
||||||
session = jsch.getSession("root", p.getStagingHost());
|
session = jsch.getSession("root", p.getHost(env));
|
||||||
System.out.println("session created.");
|
System.out.println("session created.");
|
||||||
|
|
||||||
java.util.Properties config = new java.util.Properties();
|
java.util.Properties config = new java.util.Properties();
|
||||||
|
@ -163,7 +163,7 @@ public class CheckMethod {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public void copyFromDmToSVN(File a) throws Exception {
|
public void copyFromDmToSVN(File a,String env) throws Exception {
|
||||||
JSch jsch = new JSch();
|
JSch jsch = new JSch();
|
||||||
Session session = null;
|
Session session = null;
|
||||||
ServiceConfiguration sc = new ServiceConfiguration();
|
ServiceConfiguration sc = new ServiceConfiguration();
|
||||||
|
@ -177,7 +177,7 @@ public class CheckMethod {
|
||||||
jsch.addIdentity(privateKey);
|
jsch.addIdentity(privateKey);
|
||||||
System.out.println("Private Key Added.");
|
System.out.println("Private Key Added.");
|
||||||
|
|
||||||
session = jsch.getSession("root", p.getStagingHost());
|
session = jsch.getSession("root", p.getHost(env));
|
||||||
System.out.println("session created.");
|
System.out.println("session created.");
|
||||||
|
|
||||||
java.util.Properties config = new java.util.Properties();
|
java.util.Properties config = new java.util.Properties();
|
||||||
|
@ -223,21 +223,22 @@ public class CheckMethod {
|
||||||
***REMOVED*** System.out.println(a.getStagingHost());
|
***REMOVED*** System.out.println(a.getStagingHost());
|
||||||
|
|
||||||
CheckMethod a = new CheckMethod();
|
CheckMethod a = new CheckMethod();
|
||||||
File aa = new File("OCTAVEBLACKBOX.jar");
|
***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar");
|
||||||
System.out.println(aa.getName());
|
***REMOVED*** System.out.println(aa.getName());
|
||||||
System.out.println(aa.getPath());
|
***REMOVED*** System.out.println(aa.getPath());
|
||||||
|
|
||||||
|
|
||||||
a.copyFromDmToSVN(aa);
|
|
||||||
***REMOVED*** if (a.checkMethod("dataminer1-devnext.d4science.org", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")){
|
***REMOVED***a.copyFromDmToSVN(aa);
|
||||||
***REMOVED*** System.out.println("ciao");***REMOVED***
|
if (a.checkMethod("dataminer1-devnext.d4science.org", "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548")){
|
||||||
|
System.out.println("ciaocia");***REMOVED***
|
||||||
***REMOVED******REMOVED***
|
***REMOVED******REMOVED***
|
||||||
***REMOVED*** if (a.doesExist("/home/gcube/wps_algorithms/algorithms/XMEANS_interface.jar")){
|
if (a.doesExist("/home/gcube/wps_algorithms/algorithms/RBLACKBOX.jar","Dev")){
|
||||||
***REMOVED*** System.out.println("ciao");
|
System.out.println("ciao");
|
||||||
***REMOVED******REMOVED***
|
***REMOVED******REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
***REMOVED******REMOVED***
|
***REMOVED******REMOVED***
|
||||||
|
|
||||||
***REMOVED***
|
***REMOVED******REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
|
@ -13,13 +13,21 @@ public class ClusterBuilder {
|
||||||
|
|
||||||
|
|
||||||
***REMOVED***1. to complete
|
***REMOVED***1. to complete
|
||||||
public static Cluster getStagingDataminerCluster() throws FileNotFoundException{
|
public static Cluster getStagingDataminerCluster(String env) throws FileNotFoundException{
|
||||||
Cluster cluster = new Cluster();
|
Cluster cluster = new Cluster();
|
||||||
ServiceConfiguration p = new ServiceConfiguration();
|
ServiceConfiguration p = new ServiceConfiguration();
|
||||||
Host h = new Host(p.getStagingHost());
|
Host h = new Host();
|
||||||
***REMOVED***TODO: read this from configuration or IS?
|
|
||||||
h.setName(p.getStagingHost());
|
if (env.equals("Dev")){
|
||||||
|
h.setName(p.getDevStagingHost());
|
||||||
cluster.addHost(h);
|
cluster.addHost(h);
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
if ((env.equals("Prod")||(env.equals("Proto")))){
|
||||||
|
h.setName(p.getProtoProdStagingHost());
|
||||||
|
cluster.addHost(h);
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
return cluster;
|
return cluster;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
|
@ -44,6 +44,27 @@ public class SVNUpdater {
|
||||||
***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
|
***REMOVED*** this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
|
||||||
***REMOVED******REMOVED***
|
***REMOVED******REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public String getDependencyFile(String language, String env){
|
||||||
|
|
||||||
|
String a = "";
|
||||||
|
if (env.equals("Dev")){
|
||||||
|
a= this.getDevDependencyFile(language);
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
if (env.equals("Prod")){
|
||||||
|
a= this.getProdDependencyFile(language);
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
if (env.equals("Proto")){
|
||||||
|
a= this.getRProtoDependencyFile(language);
|
||||||
|
***REMOVED***
|
||||||
|
return a;
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public String getRProtoDependencyFile(String language) {
|
public String getRProtoDependencyFile(String language) {
|
||||||
switch (language) {
|
switch (language) {
|
||||||
case "R":
|
case "R":
|
||||||
|
@ -97,6 +118,35 @@ public class SVNUpdater {
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
public String getDevDependencyFile(String language) {
|
||||||
|
switch (language) {
|
||||||
|
case "R":
|
||||||
|
return this.configuration.getSVNRDevCRANDepsList();
|
||||||
|
case "R-blackbox":
|
||||||
|
return this.configuration.getSVNRDevRBDepsList();
|
||||||
|
case "Java":
|
||||||
|
return this.configuration.getSVNRDevJavaDepsList();
|
||||||
|
case "Knime-Workflow":
|
||||||
|
return this.configuration.getSVNRDevKWDepsList();
|
||||||
|
case "Linux-compiled":
|
||||||
|
return this.configuration.getSVNRDevLinuxCompiledDepsList();
|
||||||
|
case "Octave":
|
||||||
|
return this.configuration.getSVNRDevOctaveDepsList();
|
||||||
|
case "Python":
|
||||||
|
return this.configuration.getSVNRDevPythonDepsList();
|
||||||
|
case "Pre-Installed":
|
||||||
|
return this.configuration.getSVNRDevPreInstalledDepsList();
|
||||||
|
case "Windows-compiled":
|
||||||
|
return this.configuration.getSVNRDevWCDepsList();
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
***REMOVED***
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public void readRPRotoDeps(Algorithm algorithm) throws SVNException {
|
public void readRPRotoDeps(Algorithm algorithm) throws SVNException {
|
||||||
if (algorithm.getLanguage().equals("R")) {
|
if (algorithm.getLanguage().equals("R")) {
|
||||||
|
@ -158,14 +208,62 @@ public class SVNUpdater {
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public void readRDevDeps(Algorithm algorithm) throws SVNException {
|
||||||
|
if (algorithm.getLanguage().equals("R")) {
|
||||||
|
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
if (algorithm.getLanguage().equals("R-blackbox")) {
|
||||||
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
if (algorithm.getLanguage().equals("Java")) {
|
||||||
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
if (algorithm.getLanguage().equals("Knime-Workflow")) {
|
||||||
|
this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
if (algorithm.getLanguage().equals("Linux-compiled")) {
|
||||||
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
if (algorithm.getLanguage().equals("Octave")) {
|
||||||
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
if (algorithm.getLanguage().equals("Python")) {
|
||||||
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
if (algorithm.getLanguage().equals("Windows-compiled")) {
|
||||||
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
if (algorithm.getLanguage().equals("Pre-Installed")) {
|
||||||
|
this.checkIfAvaialable("", algorithm.getDependencies());
|
||||||
|
***REMOVED***
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
***REMOVED*** public void updateProdDeps(Algorithm algorithm) {
|
***REMOVED*** public void updateProdDeps(Algorithm algorithm) {
|
||||||
***REMOVED*** this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
|
***REMOVED*** this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
|
||||||
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
|
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
|
||||||
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
|
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
|
||||||
***REMOVED******REMOVED***
|
***REMOVED******REMOVED***
|
||||||
|
|
||||||
public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) {
|
public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) {
|
||||||
|
|
||||||
|
if (env.equals("Dev")){
|
||||||
|
this.updateSVNAlgorithmList(this.configuration.getSVNDevAlgorithmsList(), algorithm, targetVRE, user, env);
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
if (env.equals("Prod")){
|
||||||
|
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env);
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
if (env.equals("Proto")){
|
||||||
this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env);
|
this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env);
|
||||||
|
***REMOVED***
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) {
|
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env) {
|
||||||
|
@ -350,10 +448,11 @@ public class SVNUpdater {
|
||||||
System.out.println("Checking dependencies list: " + file);
|
System.out.println("Checking dependencies list: " + file);
|
||||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
|
||||||
List<String> validDependencies = Arrays.asList(lines);
|
|
||||||
|
|
||||||
|
|
||||||
|
List<String> validDependencies = new LinkedList<String>();
|
||||||
|
for(String l: byteArrayOutputStream.toString().split("\\r?\\n")){
|
||||||
|
validDependencies.add(l.trim());
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
List<String> undefined = new LinkedList<String>();
|
List<String> undefined = new LinkedList<String>();
|
||||||
|
|
||||||
|
|
|
@ -133,6 +133,49 @@ public class ServiceConfiguration {
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
***REMOVED***dev
|
||||||
|
public String getSVNDevAlgorithmsList(){
|
||||||
|
return props.getProperty("svn.dev.algorithms-list");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevLinuxCompiledDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-linux-compiled");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevCRANDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-r");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevPreInstalledDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-pre-installed");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevRBDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-r-blackbox");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevJavaDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-java");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevKWDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-knime-workflow");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevOctaveDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-octave");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevPythonDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-python");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
public String getSVNRDevWCDepsList(){
|
||||||
|
return props.getProperty("svn.dev.deps-windows-compiled");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public String getCSVUrl() {
|
public String getCSVUrl() {
|
||||||
|
@ -140,15 +183,42 @@ public class ServiceConfiguration {
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
public String getStagingHost() {
|
public String getHost(String env){
|
||||||
return props.getProperty("STAGING_HOST");
|
String a = "";
|
||||||
|
|
||||||
|
if (env.equals("Dev")){
|
||||||
|
a = this.getDevStagingHost();
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
if (env.equals("Prod")||(env.equals("Prod"))){
|
||||||
|
a = this.getProtoProdStagingHost();
|
||||||
|
***REMOVED***
|
||||||
|
return a;
|
||||||
|
|
||||||
|
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
public String getDevStagingHost() {
|
||||||
|
return props.getProperty("DEV_STAGING_HOST");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
public String getProtoProdStagingHost() {
|
||||||
|
return props.getProperty("PROTO_PROD_STAGING_HOST");
|
||||||
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
public static void main(String[] args) throws FileNotFoundException {
|
public static void main(String[] args) throws FileNotFoundException {
|
||||||
ServiceConfiguration a = new ServiceConfiguration();
|
ServiceConfiguration a = new ServiceConfiguration();
|
||||||
System.out.println(a.getStagingHost());
|
***REMOVED***System.out.println(a.getStagingHost());
|
||||||
|
System.out.println(a.getDevStagingHost());
|
||||||
|
System.out.println(a.getProtoProdStagingHost());
|
||||||
System.out.println(a.getCSVUrl());
|
System.out.println(a.getCSVUrl());
|
||||||
System.out.println(a.getSVNMainAlgoRepo());
|
System.out.println(a.getSVNMainAlgoRepo());
|
||||||
|
System.out.println(a.getSVNRProtoCRANDepsList());
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
#YML node file
|
#YML node file
|
||||||
#STAGING_HOST: dataminer-proto-ghost.d4science.org
|
DEV_STAGING_HOST: dataminer1-devnext.d4science.org
|
||||||
STAGING_HOST: dataminer1-devnext.d4science.org
|
PROTO_PROD_STAGING_HOST: dataminer-proto-ghost.d4science.org
|
||||||
SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
|
SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
|
||||||
#HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0
|
#HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0
|
||||||
|
|
||||||
|
@ -34,3 +34,17 @@ svn.prod.deps-octave =
|
||||||
svn.prod.deps-python =
|
svn.prod.deps-python =
|
||||||
svn.prod.deps-windows-compiled =
|
svn.prod.deps-windows-compiled =
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
svn.dev.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/dev/algorithms
|
||||||
|
|
||||||
|
svn.dev.deps-linux-compiled =
|
||||||
|
svn.dev.deps-pre-installed =
|
||||||
|
svn.dev.deps-r-blackbox =
|
||||||
|
svn.dev.deps-r = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
|
||||||
|
svn.dev.deps-java =
|
||||||
|
svn.dev.deps-knime-workflow =
|
||||||
|
svn.dev.deps-octave =
|
||||||
|
svn.dev.deps-python =
|
||||||
|
svn.dev.deps-windows-compiled =
|
||||||
|
|
||||||
|
|
|
@ -1,7 +1,7 @@
|
||||||
<application mode="online">
|
<application mode="online">
|
||||||
|
|
||||||
<name>dataminer-pool-manager</name>
|
<name>dataminer-pool-manager</name>
|
||||||
<group>dataanalysis</group>
|
<group>DataAnalysis</group>
|
||||||
<version>0.0.1</version>
|
<version>0.0.1</version>
|
||||||
|
|
||||||
<!--<description>Lorem ipsum dolor sit amet...</description>-->
|
<!--<description>Lorem ipsum dolor sit amet...</description>-->
|
||||||
|
|
|
@ -5,6 +5,12 @@
|
||||||
<web-app>
|
<web-app>
|
||||||
<display-name>Archetype Created Web Application</display-name>
|
<display-name>Archetype Created Web Application</display-name>
|
||||||
|
|
||||||
|
<context-param>
|
||||||
|
<param-name>Environment</param-name>
|
||||||
|
<param-value>Dev</param-value>
|
||||||
|
<!-- Possible Values: Dev, Proto, Prod -->
|
||||||
|
<description>This is a context parameter example</description>
|
||||||
|
</context-param>
|
||||||
|
|
||||||
<servlet>
|
<servlet>
|
||||||
<servlet-name>REST-API</servlet-name>
|
<servlet-name>REST-API</servlet-name>
|
||||||
|
@ -16,27 +22,37 @@
|
||||||
org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
|
org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
|
||||||
</init-param>
|
</init-param>
|
||||||
<load-on-startup>1</load-on-startup>
|
<load-on-startup>1</load-on-startup>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</servlet>
|
</servlet>
|
||||||
|
|
||||||
<servlet-mapping>
|
<servlet-mapping>
|
||||||
<servlet-name>REST-API</servlet-name>
|
<servlet-name>REST-API</servlet-name>
|
||||||
<url-pattern>/api/*</url-pattern>
|
<url-pattern>/api/*</url-pattern>
|
||||||
|
|
||||||
</servlet-mapping>
|
</servlet-mapping>
|
||||||
|
|
||||||
<!--
|
|
||||||
<servlet>
|
|
||||||
<servlet-name>Jersey2Config</servlet-name>
|
|
||||||
<servlet-class>io.swagger.jersey.config.JerseyJaxrsConfig</servlet-class>
|
<!-- <servlet> <servlet-name>Jersey2Config</servlet-name> <servlet-class>io.swagger.jersey.config.JerseyJaxrsConfig</servlet-class>
|
||||||
<init-param>
|
<init-param> <param-name>api.version</param-name> <param-value>1.0.0</param-value>
|
||||||
<param-name>api.version</param-name>
|
</init-param> <init-param> <param-name>swagger.api.basepath</param-name>
|
||||||
<param-value>1.0.0</param-value>
|
<param-value>http:***REMOVED***localhost:8080/api</param-value> </init-param> <load-on-startup>2</load-on-startup>
|
||||||
</init-param>
|
</servlet> -->
|
||||||
<init-param>
|
|
||||||
<param-name>swagger.api.basepath</param-name>
|
|
||||||
<param-value>http:***REMOVED***localhost:8080/api</param-value>
|
|
||||||
</init-param>
|
<!-- <security-constraint> <web-resource-collection> <web-resource-name>Viewpoint
|
||||||
<load-on-startup>2</load-on-startup>
|
Secure URLs</web-resource-name> <url-pattern>/api/*</url-pattern> </web-resource-collection>
|
||||||
</servlet>
|
<user-data-constraint> <transport-guarantee>CONFIDENTIAL</transport-guarantee>
|
||||||
-->
|
</user-data-constraint> </security-constraint> -->
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
</web-app>
|
</web-app>
|
||||||
|
|
|
@ -24,10 +24,10 @@ public class JobTest {
|
||||||
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0");
|
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0");
|
||||||
|
|
||||||
***REMOVED***test phase
|
***REMOVED***test phase
|
||||||
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
|
***REMOVED***Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
|
||||||
***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
|
***REMOVED***Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
|
||||||
DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ ScopeProvider.instance.get());
|
***REMOVED***DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, /*rProtoCluster,*/ ScopeProvider.instance.get());
|
||||||
job.start();
|
***REMOVED***job.start();
|
||||||
|
|
||||||
***REMOVED***release phase
|
***REMOVED***release phase
|
||||||
***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
|
***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
|
||||||
|
|
Loading…
Reference in New Issue