git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@157787 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Nunzio Andrea Galante
parent
3057697992
commit
1d7e77a3c2
|
|
@ -8,4 +8,7 @@
|
|||
<Changeset component="dataminer-pool-manager.2-1-0" date="${buildDate}">
|
||||
<Change>New configuration file added</Change>
|
||||
</Changeset>
|
||||
<Changeset component="dataminer-pool-manager.2-1-1" date="${buildDate}">
|
||||
<Change>Improvements and bugs fixed/SVN-UTF8 compliant/Installation in Production ghost added</Change>
|
||||
</Changeset>
|
||||
</ReleaseNotes>
|
||||
2
pom.xml
2
pom.xml
|
|
@ -12,7 +12,7 @@
|
|||
<groupId>org.gcube.dataAnalysis</groupId>
|
||||
<artifactId>dataminer-pool-manager</artifactId>
|
||||
<packaging>war</packaging>
|
||||
<version>2.1.0-SNAPSHOT</version>
|
||||
<version>2.1.1-SNAPSHOT</version>
|
||||
|
||||
<name>dataminer-pool-manager</name>
|
||||
<description>
|
||||
|
|
|
|||
|
|
@ -42,9 +42,9 @@ public class DataminerPoolManager {
|
|||
|
||||
public String publishAlgorithm(Algorithm algo, String targetVRE, String category, String algorithm_type/*, String env*/) throws IOException, InterruptedException {
|
||||
|
||||
//Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
|
||||
Cluster prodCluster = ClusterBuilder.getProdDataminerCluster();
|
||||
|
||||
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, /*prodCluster,*/ targetVRE, category, algorithm_type/*,env*/);
|
||||
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/);
|
||||
String id = job.start();
|
||||
return id;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,20 +4,23 @@ import java.io.File;
|
|||
import java.io.FileNotFoundException;
|
||||
import java.io.PrintWriter;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.util.Collection;
|
||||
|
||||
import org.gcube.common.authorization.library.provider.SecurityTokenProvider;
|
||||
import org.gcube.common.scope.api.ScopeProvider;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckPermission;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
|
||||
|
||||
public class ProductionPublishingJob extends DMPMJob {
|
||||
|
||||
private Algorithm algorithm;
|
||||
//private Cluster prodCluster;
|
||||
private Cluster prodCluster;
|
||||
private String targetVREName;
|
||||
private String category;
|
||||
private String algorithm_type;
|
||||
|
|
@ -26,41 +29,81 @@ public class ProductionPublishingJob extends DMPMJob {
|
|||
|
||||
|
||||
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm,
|
||||
/*Cluster prodCluster,*/ String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException {
|
||||
Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException {
|
||||
super(svnUpdater);
|
||||
this.jobLogs = new File(
|
||||
System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
|
||||
this.jobLogs.mkdirs();
|
||||
this.algorithm = algorithm;
|
||||
//this.prodCluster = prodCluster;
|
||||
this.prodCluster = prodCluster;
|
||||
this.targetVREName = targetVREName;
|
||||
this.category = category;
|
||||
this.algorithm_type = algorithm_type;
|
||||
//this.targetVREToken = targetVREToken;
|
||||
//this.env= env;
|
||||
|
||||
|
||||
this.jobLogs = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
|
||||
this.jobLogs.mkdirs();
|
||||
this.getStatus(0);
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void execute() {
|
||||
SendMail sm = new SendMail();
|
||||
NotificationHelper nh = new NotificationHelper();
|
||||
ServiceConfiguration a = new ServiceConfiguration();
|
||||
CheckMethod b = new CheckMethod();
|
||||
try {
|
||||
|
||||
//if (CheckPermission.apply(targetVREToken,targetVREName)){
|
||||
|
||||
//this.svnUpdater.updateProdDeps(this.algorithm);
|
||||
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, this.category,this.algorithm_type, this.algorithm.getFullname()/*, env*/);
|
||||
this.getStatus(9);
|
||||
sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBodyRelease("\n\n"+this.buildInfo()));
|
||||
|
||||
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
|
||||
this.svnUpdater.getDependencyFileProd(this.algorithm.getLanguage()/*,env*/),
|
||||
this.algorithm.getDependencies());
|
||||
|
||||
if (!undefinedDependencies.isEmpty()) {
|
||||
|
||||
String message = "Following dependencies are not defined:\n";
|
||||
for (String n : undefinedDependencies) {
|
||||
message += "\n" + n +"\n";
|
||||
}
|
||||
this.getStatus(2);
|
||||
|
||||
sm.sendNotification(nh.getFailedSubjectRelease() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo()));
|
||||
return;
|
||||
//}
|
||||
//else this.getStatus(0);
|
||||
//sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBodyRelease(" The user "+this.algorithm.getFullname()+ " is not authorized to access to the "+ targetVREName+ " VRE"+"\n\n"+this.buildInfo()));
|
||||
//return;
|
||||
// int ret = this.executeAnsibleWorker(
|
||||
// createWorker(this.algorithm, this.prodCluster, false, "gcube"));
|
||||
|
||||
}
|
||||
b.deleteFilesProd(this.algorithm);
|
||||
|
||||
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.prodCluster, false, "root"));
|
||||
|
||||
if (ret != 0) {
|
||||
this.getStatus(2);
|
||||
sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo());
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
if (ret == 0) {
|
||||
this.getStatus(0);
|
||||
|
||||
if (b.checkMethod(a.getProdHost(), SecurityTokenProvider.instance.get())&&(b.algoExistsProd(this.algorithm/*, env*/))) {
|
||||
|
||||
System.out.println("Interface check ok!");
|
||||
System.out.println("Both the files exist at the correct path!");
|
||||
|
||||
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName,this.category, this.algorithm_type,
|
||||
this.algorithm.getFullname()/*, env*/);
|
||||
|
||||
this.getStatus(9);
|
||||
sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
|
||||
return;
|
||||
} else
|
||||
this.getStatus(2);
|
||||
sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm",
|
||||
nh.getFailedBody(
|
||||
"\n"+
|
||||
"Installation completed but DataMiner Interface not working correctly or files "
|
||||
+ this.algorithm.getName() + ".jar and " + this.algorithm.getName()
|
||||
+ "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
|
||||
return;
|
||||
|
||||
}
|
||||
} catch (Exception e) {
|
||||
try {
|
||||
this.getStatus(0);
|
||||
|
|
@ -75,42 +118,51 @@ public class ProductionPublishingJob extends DMPMJob {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException {
|
||||
|
||||
File m = new File(this.jobLogs + File.separator + this.id+"_exitStatus");
|
||||
File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
|
||||
PrintWriter writer = new PrintWriter(m, "UTF-8");
|
||||
|
||||
String response = "";
|
||||
|
||||
if (exitstatus == 0) {
|
||||
response = "IN PROGRESS";
|
||||
writer.println(response);
|
||||
//writer.close();
|
||||
}
|
||||
|
||||
if (exitstatus == 9) {
|
||||
response = "COMPLETED";
|
||||
writer.println(response);
|
||||
//writer.close();
|
||||
}
|
||||
|
||||
if (exitstatus == 0) {
|
||||
|
||||
if (exitstatus == 2) {
|
||||
response = "FAILED";
|
||||
writer.println(response);
|
||||
//writer.close();
|
||||
}
|
||||
|
||||
|
||||
writer.close();
|
||||
return response;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public String buildInfo(){
|
||||
public String buildInfo() throws UnsupportedEncodingException{
|
||||
ServiceConfiguration a = new ServiceConfiguration();
|
||||
|
||||
return
|
||||
"\n"+
|
||||
"Algorithm details:\n"+"\n"+
|
||||
"User: "+this.algorithm.getFullname()+"\n"+
|
||||
"Algorithm name: "+this.algorithm.getName()+"\n"+
|
||||
"Staging DataMiner Host: "+ a.getProdHost()+"\n"+
|
||||
"Caller VRE: "+ScopeProvider.instance.get()+"\n"+
|
||||
"Target VRE: "+targetVREName+"\n";
|
||||
"Target VRE: "+this.targetVREName+"\n";
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -87,7 +87,6 @@ public class CheckMethod {
|
|||
}
|
||||
|
||||
|
||||
|
||||
public boolean algoExists(Algorithm a/*, String env*/) throws Exception{
|
||||
ServiceConfiguration p = new ServiceConfiguration();
|
||||
|
||||
|
|
@ -112,6 +111,30 @@ public class CheckMethod {
|
|||
}
|
||||
|
||||
|
||||
public boolean algoExistsProd(Algorithm a/*, String env*/) throws Exception{
|
||||
ServiceConfiguration p = new ServiceConfiguration();
|
||||
|
||||
File file = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+".jar");
|
||||
File file2 = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
|
||||
|
||||
|
||||
System.out.println("First file is located to: "+file.getPath());
|
||||
System.out.println("Second file is located to: "+file2.getPath());
|
||||
|
||||
|
||||
if ((this.doesExistProd(file.getPath()/*,env*/)) && (this.doesExistProd(file2.getPath()/*,env*/))){
|
||||
this.copyFromDmToSVNProd(file/*,env*/);
|
||||
this.copyFromDmToSVNProd(file2/*,env*/);
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
else
|
||||
System.out.println("Algorithm "+a.getName()+".jar"+ " and "+a.getName()+"_interface.jar files are not present at the expected path");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
public void deleteFiles(Algorithm a/*,String env*/) throws Exception{
|
||||
JSch jsch = new JSch();
|
||||
Session session = null;
|
||||
|
|
@ -163,7 +186,55 @@ public class CheckMethod {
|
|||
}
|
||||
|
||||
|
||||
|
||||
public void deleteFilesProd(Algorithm a/*,String env*/) throws Exception{
|
||||
JSch jsch = new JSch();
|
||||
Session session = null;
|
||||
Channel channel = null;
|
||||
ChannelSftp c = null;
|
||||
ServiceConfiguration p = new ServiceConfiguration();
|
||||
System.out.println("checking existing in env: " + p.getProdHost());
|
||||
|
||||
File file = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+".jar");
|
||||
File file2 = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
|
||||
|
||||
|
||||
System.out.println("First file is located to: "+file.getPath());
|
||||
System.out.println("Second file is located to: "+file2.getPath());
|
||||
|
||||
|
||||
jsch.setKnownHosts("~/.ssh/known_hosts");
|
||||
String privateKey = "~/.ssh/id_rsa";
|
||||
|
||||
jsch.addIdentity(privateKey);
|
||||
System.out.println("Private Key Added.");
|
||||
|
||||
session = jsch.getSession("root", p.getProdHost());
|
||||
System.out.println("session created.");
|
||||
|
||||
java.util.Properties config = new java.util.Properties();
|
||||
config.put("StrictHostKeyChecking", "no");
|
||||
|
||||
session.setConfig(config);
|
||||
session.connect();
|
||||
|
||||
channel = session.openChannel("sftp");
|
||||
channel.connect();
|
||||
System.out.println("shell channel connected....");
|
||||
|
||||
c = (ChannelSftp) channel;
|
||||
|
||||
if(doesExistProd(file.getPath()/*,env*/)&&(doesExistProd(file2.getPath()/*,env*/))){
|
||||
|
||||
c.rm(file.getPath());
|
||||
c.rm(file2.getPath());
|
||||
System.out.println("Both the files have been deleted");
|
||||
}
|
||||
else System.out.println("Files not found");
|
||||
channel.disconnect();
|
||||
c.disconnect();
|
||||
session.disconnect();
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
@ -219,6 +290,55 @@ public class CheckMethod {
|
|||
|
||||
|
||||
|
||||
public boolean doesExistProd(String path/*, String env*/) throws Exception {
|
||||
JSch jsch = new JSch();
|
||||
Session session = null;
|
||||
Channel channel = null;
|
||||
ChannelSftp c = null;
|
||||
boolean success = false;
|
||||
ServiceConfiguration p = new ServiceConfiguration();
|
||||
|
||||
|
||||
jsch.setKnownHosts("~/.ssh/known_hosts");
|
||||
String privateKey = "~/.ssh/id_rsa";
|
||||
|
||||
jsch.addIdentity(privateKey);
|
||||
System.out.println("Private Key Added.");
|
||||
|
||||
session = jsch.getSession("root", p.getProdHost());
|
||||
System.out.println("session created.");
|
||||
|
||||
java.util.Properties config = new java.util.Properties();
|
||||
config.put("StrictHostKeyChecking", "no");
|
||||
|
||||
session.setConfig(config);
|
||||
session.connect();
|
||||
|
||||
channel = session.openChannel("sftp");
|
||||
channel.connect();
|
||||
System.out.println("shell channel connected....");
|
||||
|
||||
c = (ChannelSftp) channel;
|
||||
|
||||
SftpATTRS is = null;
|
||||
System.out.println(path);
|
||||
|
||||
try {
|
||||
is = c.lstat(path);
|
||||
success = true;
|
||||
} catch (SftpException e) {
|
||||
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
|
||||
// file doesn't exist
|
||||
success = false;
|
||||
}
|
||||
//success = true; // something else went wrong
|
||||
}
|
||||
channel.disconnect();
|
||||
c.disconnect();
|
||||
session.disconnect();
|
||||
return success;
|
||||
|
||||
}
|
||||
|
||||
|
||||
public void copyFromDmToSVN(File a/*,String env*/) throws Exception {
|
||||
|
|
@ -264,9 +384,50 @@ public class CheckMethod {
|
|||
f.delete();
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void copyFromDmToSVNProd(File a/*,String env*/) throws Exception {
|
||||
JSch jsch = new JSch();
|
||||
Session session = null;
|
||||
ServiceConfiguration sc = new ServiceConfiguration();
|
||||
SVNUpdater svnUpdater = new SVNUpdater(sc);
|
||||
|
||||
ServiceConfiguration p = new ServiceConfiguration();
|
||||
|
||||
jsch.setKnownHosts("~/.ssh/known_hosts");
|
||||
String privateKey = "~/.ssh/id_rsa";
|
||||
|
||||
jsch.addIdentity(privateKey);
|
||||
System.out.println("Private Key Added.");
|
||||
|
||||
session = jsch.getSession("root", p.getProdHost());
|
||||
System.out.println("session created.");
|
||||
|
||||
java.util.Properties config = new java.util.Properties();
|
||||
config.put("StrictHostKeyChecking", "no");
|
||||
|
||||
session.setConfig(config);
|
||||
session.connect();
|
||||
|
||||
Channel channel = session.openChannel("sftp");
|
||||
channel.connect();
|
||||
|
||||
ChannelSftp sftp = (ChannelSftp) channel;
|
||||
|
||||
sftp.cd(p.getProdGhostAlgoDirectory());
|
||||
|
||||
System.out.println("REMOTE : "+p.getProdGhostAlgoDirectory()+"/"+a.getName());
|
||||
System.out.println("LOCAL : /tmp/"+a.getName());
|
||||
|
||||
sftp.get(p.getProdGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
|
||||
|
||||
channel.disconnect();
|
||||
session.disconnect();
|
||||
|
||||
|
||||
File f = new File("/tmp/"+a.getName());
|
||||
svnUpdater.updateAlgorithmFilesProd(f);
|
||||
f.delete();
|
||||
}
|
||||
|
||||
public List<String> getFiles(String a){
|
||||
|
||||
|
|
@ -281,13 +442,6 @@ public class CheckMethod {
|
|||
return ls;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
// ServiceConfiguration a = new ServiceConfiguration();
|
||||
|
|
@ -295,7 +449,7 @@ public class CheckMethod {
|
|||
|
||||
CheckMethod a = new CheckMethod();
|
||||
|
||||
a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
|
||||
//a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
|
||||
|
||||
// File aa = new File("OCTAVEBLACKBOX.jar");
|
||||
// System.out.println(aa.getName());
|
||||
|
|
@ -321,8 +475,8 @@ public class CheckMethod {
|
|||
// a.copyFromDmToSVN(aa, "Dev");
|
||||
|
||||
|
||||
//a.checkMethod("dataminer-ghost-d.dev.d4science.org",
|
||||
// "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
|
||||
System.out.println(a.checkMethod("dm-pool-manager-proto.d4science.org",
|
||||
"2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548"));
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,29 @@ public class ClusterBuilder {
|
|||
return cluster;
|
||||
}
|
||||
|
||||
|
||||
public static Cluster getProdDataminerCluster() throws FileNotFoundException{
|
||||
Cluster cluster = new Cluster();
|
||||
ServiceConfiguration p = new ServiceConfiguration();
|
||||
Host h = new Host();
|
||||
h.setName(p.getProdHost());
|
||||
cluster.addHost(h);
|
||||
|
||||
// if (env.equals("Dev")){
|
||||
// h.setName(p.getDevStagingHost());
|
||||
// cluster.addHost(h);
|
||||
// }
|
||||
//
|
||||
// if ((env.equals("Prod")||(env.equals("Proto")))){
|
||||
// h.setName(p.getProtoProdStagingHost());
|
||||
// cluster.addHost(h);
|
||||
// }
|
||||
|
||||
return cluster;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException{
|
||||
Cluster cluster = new Cluster();
|
||||
for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) {
|
||||
|
|
|
|||
|
|
@ -15,11 +15,11 @@ public class NotificationHelper {
|
|||
}
|
||||
|
||||
private String getSubjectHeader() {
|
||||
return "[DataMinerGhostInstallationRequestReport]";
|
||||
return "[DataMinerGhostStagingInstallationRequestReport]";
|
||||
}
|
||||
|
||||
private String getSubjectHeaderRelease() {
|
||||
return "[DataMinerReleaseInstallationRequestReport]";
|
||||
return "[DataMinerGhostProductionInstallationRequestReport]";
|
||||
}
|
||||
|
||||
private boolean isError() {
|
||||
|
|
@ -49,7 +49,7 @@ public class NotificationHelper {
|
|||
}
|
||||
|
||||
public String getSuccessBody(String info) {
|
||||
String message = String.format("The installation of the algorithm in the ghost dataminer is completed successfully.");
|
||||
String message = String.format("The installation of the algorithm is completed successfully.");
|
||||
message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info;
|
||||
return message;
|
||||
}
|
||||
|
|
@ -60,17 +60,17 @@ public class NotificationHelper {
|
|||
return body;
|
||||
}
|
||||
|
||||
public String getSuccessBodyRelease(String info) {
|
||||
String message = String.format("SVN REPOSITORY CORRECTLY UPDATED.");
|
||||
message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info;
|
||||
return message;
|
||||
}
|
||||
|
||||
public String getFailedBodyRelease(String info) {
|
||||
String message = String.format("SVN REPOSITORY UPDATE FAILED.");
|
||||
message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info;
|
||||
return message;
|
||||
}
|
||||
// public String getSuccessBodyRelease(String info) {
|
||||
// String message = String.format("SVN REPOSITORY CORRECTLY UPDATED.");
|
||||
// message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info;
|
||||
// return message;
|
||||
// }
|
||||
//
|
||||
// public String getFailedBodyRelease(String info) {
|
||||
// String message = String.format("SVN REPOSITORY UPDATE FAILED.");
|
||||
// message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info;
|
||||
// return message;
|
||||
// }
|
||||
|
||||
// public String getSubject() {
|
||||
// if(this.isError()) {
|
||||
|
|
|
|||
|
|
@ -19,14 +19,12 @@ import java.io.ByteArrayInputStream;
|
|||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
import java.sql.Timestamp;
|
||||
import java.io.UnsupportedEncodingException;
|
||||
import java.text.DateFormat;
|
||||
import java.text.ParseException;
|
||||
import java.text.SimpleDateFormat;
|
||||
import java.util.*;
|
||||
|
||||
import javax.ws.rs.core.NewCookie;
|
||||
|
||||
/**
|
||||
* Created by ggiammat on 5/9/17.
|
||||
*/
|
||||
|
|
@ -72,116 +70,32 @@ public class SVNUpdater {
|
|||
return a;
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
public String getDependencyFileProd(String language/*, String env*/){
|
||||
|
||||
String a = "";
|
||||
// if (env.equals("Dev")){
|
||||
// a= this.getDevDependencyFile(language);
|
||||
// }
|
||||
//
|
||||
// if (env.equals("Prod")){
|
||||
// a= this.getProdDependencyFile(language);
|
||||
// }
|
||||
//
|
||||
// if (env.equals("Proto")){
|
||||
// a= this.getRProtoDependencyFile(language);
|
||||
// }
|
||||
//
|
||||
// if (env.equals("Preprod")){
|
||||
// a= this.getPreProdDependencyFile(language);
|
||||
// }
|
||||
a = this.getProdDependencyFile(language);
|
||||
return a;
|
||||
}
|
||||
|
||||
|
||||
// public String getRProtoDependencyFile(String language) {
|
||||
// switch (language) {
|
||||
// case "R":
|
||||
// return this.configuration.getSVNRProtoCRANDepsList();
|
||||
// case "R-blackbox":
|
||||
// return this.configuration.getSVNRProtoRBDepsList();
|
||||
// case "Java":
|
||||
// return this.configuration.getSVNRProtoJavaDepsList();
|
||||
// case "Knime-Workflow":
|
||||
// return this.configuration.getSVNRProtoKWDepsList();
|
||||
// case "Linux-compiled":
|
||||
// return this.configuration.getSVNRProtoLinuxCompiledDepsList();
|
||||
// case "Octave":
|
||||
// return this.configuration.getSVNRProtoOctaveDepsList();
|
||||
// case "Python":
|
||||
// return this.configuration.getSVNRProtoPythonDepsList();
|
||||
// case "Pre-Installed":
|
||||
// return this.configuration.getSVNRProtoPreInstalledDepsList();
|
||||
// case "Windows-compiled":
|
||||
// return this.configuration.getSVNRProtoWCDepsList();
|
||||
// default:
|
||||
// return null;
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
// public String getPreProdDependencyFile(String language) {
|
||||
// switch (language) {
|
||||
// case "R":
|
||||
// return this.configuration.getSVNPreProdCRANDepsList();
|
||||
// case "R-blackbox":
|
||||
// return this.configuration.getSVNPreProdRBDepsList();
|
||||
// case "Java":
|
||||
// return this.configuration.getSVNPreProdJavaDepsList();
|
||||
// case "Knime-Workflow":
|
||||
// return this.configuration.getSVNPreProdKWDepsList();
|
||||
// case "Linux-compiled":
|
||||
// return this.configuration.getSVNPreProdLinuxCompiledDepsList();
|
||||
// case "Octave":
|
||||
// return this.configuration.getSVNPreProdOctaveDepsList();
|
||||
// case "Python":
|
||||
// return this.configuration.getSVNPreProdPythonDepsList();
|
||||
// case "Pre-Installed":
|
||||
// return this.configuration.getSVNPreProdPreInstalledDepsList();
|
||||
// case "Windows-compiled":
|
||||
// return this.configuration.getSVNPreProdWCDepsList();
|
||||
// default:
|
||||
// return null;
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// public String getProdDependencyFile(String language) {
|
||||
// switch (language) {
|
||||
// case "R":
|
||||
// return this.configuration.getSVNProdCRANDepsList();
|
||||
// case "R-blackbox":
|
||||
// return this.configuration.getSVNProdRBDepsList();
|
||||
// case "Java":
|
||||
// return this.configuration.getSVNProdJavaDepsList();
|
||||
// case "Knime-Workflow":
|
||||
// return this.configuration.getSVNProdKWDepsList();
|
||||
// case "Linux-compiled":
|
||||
// return this.configuration.getSVNProdLinuxCompiledDepsList();
|
||||
// case "Octave":
|
||||
// return this.configuration.getSVNProdOctaveDepsList();
|
||||
// case "Python":
|
||||
// return this.configuration.getSVNProdPythonDepsList();
|
||||
// case "Pre-Installed":
|
||||
// return this.configuration.getSVNProdPreInstalledDepsList();
|
||||
// case "Windows-compiled":
|
||||
// return this.configuration.getSVNProdWCDepsList();
|
||||
// default:
|
||||
// return null;
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
// public String getDevDependencyFile(String language) {
|
||||
// switch (language) {
|
||||
// case "R":
|
||||
// return this.configuration.getSVNDevCRANDepsList();
|
||||
// case "R-blackbox":
|
||||
// return this.configuration.getSVNDevRBDepsList();
|
||||
// case "Java":
|
||||
// return this.configuration.getSVNDevJavaDepsList();
|
||||
// case "Knime-Workflow":
|
||||
// return this.configuration.getSVNDevKWDepsList();
|
||||
// case "Linux-compiled":
|
||||
// return this.configuration.getSVNDevLinuxCompiledDepsList();
|
||||
// case "Octave":
|
||||
// return this.configuration.getSVNDevOctaveDepsList();
|
||||
// case "Python":
|
||||
// return this.configuration.getSVNDevPythonDepsList();
|
||||
// case "Pre-Installed":
|
||||
// return this.configuration.getSVNDevPreInstalledDepsList();
|
||||
// case "Windows-compiled":
|
||||
// return this.configuration.getSVNDevWCDepsList();
|
||||
// default:
|
||||
// return null;
|
||||
// }
|
||||
// }
|
||||
//
|
||||
|
||||
|
||||
public String getStagingDependencyFile(String language) {
|
||||
switch (language) {
|
||||
|
|
@ -209,162 +123,32 @@ public class SVNUpdater {
|
|||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
// public void readPreProdDeps(Algorithm algorithm) throws SVNException {
|
||||
// if (algorithm.getLanguage().equals("R")) {
|
||||
// this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("R-blackbox")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Java")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Knime-Workflow")) {
|
||||
// this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Linux-compiled")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Octave")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Python")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Windows-compiled")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Pre-Installed")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
//
|
||||
// public void readRPRotoDeps(Algorithm algorithm) throws SVNException {
|
||||
// if (algorithm.getLanguage().equals("R")) {
|
||||
// this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("R-blackbox")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Java")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Knime-Workflow")) {
|
||||
// this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Linux-compiled")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Octave")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Python")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Windows-compiled")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Pre-Installed")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// public void readProdDeps(Algorithm algorithm) throws SVNException {
|
||||
// if (algorithm.getLanguage().equals("R")) {
|
||||
// this.checkIfAvaialable(this.configuration.getSVNProdCRANDepsList(), algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("R-blackbox")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Java")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Knime-Workflow")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Linux-compiled")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Octave")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Python")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Windows-compiled")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Pre-Installed")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
//
|
||||
// public void readRDevDeps(Algorithm algorithm) throws SVNException {
|
||||
// if (algorithm.getLanguage().equals("R")) {
|
||||
// this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("R-blackbox")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Java")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Knime-Workflow")) {
|
||||
// this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Linux-compiled")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Octave")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Python")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Windows-compiled")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// if (algorithm.getLanguage().equals("Pre-Installed")) {
|
||||
// this.checkIfAvaialable("", algorithm.getDependencies());
|
||||
// }
|
||||
// }
|
||||
|
||||
|
||||
|
||||
|
||||
public String getProdDependencyFile(String language) {
|
||||
switch (language) {
|
||||
case "R":
|
||||
return this.configuration.getSVNProdCRANDepsList();
|
||||
case "R-blackbox":
|
||||
return this.configuration.getSVNProdRBDepsList();
|
||||
case "Java":
|
||||
return this.configuration.getSVNProdJavaDepsList();
|
||||
case "Knime-Workflow":
|
||||
return this.configuration.getSVNProdKWDepsList();
|
||||
case "Linux-compiled":
|
||||
return this.configuration.getSVNProdLinuxCompiledDepsList();
|
||||
case "Octave":
|
||||
return this.configuration.getSVNProdOctaveDepsList();
|
||||
case "Python":
|
||||
return this.configuration.getSVNProdPythonDepsList();
|
||||
case "Pre-Installed":
|
||||
return this.configuration.getSVNProdPreInstalledDepsList();
|
||||
case "Windows-compiled":
|
||||
return this.configuration.getSVNProdWCDepsList();
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// public void updateProdDeps(Algorithm algorithm) {
|
||||
// this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
|
||||
// this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
|
||||
// this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
|
||||
// }
|
||||
|
||||
// public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
|
||||
//
|
||||
// if (env.equals("Dev")){
|
||||
// this.updateSVNAlgorithmList(this.configuration.getSVNDevAlgorithmsList(), algorithm, targetVRE, category,algorithm_type, user, env);
|
||||
// }
|
||||
//
|
||||
// if (env.equals("Prod")){
|
||||
// this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
|
||||
// }
|
||||
//
|
||||
// if (env.equals("Proto")){
|
||||
// this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
|
||||
// }
|
||||
//
|
||||
// if (env.equals("Preprod")){
|
||||
// this.updateSVNAlgorithmList(this.configuration.getSVNPreProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
|
||||
// }
|
||||
// }
|
||||
|
||||
public void updateSVNStagingAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
|
||||
this.updateSVNAlgorithmList(this.configuration.getSVNStagingAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
|
||||
|
|
@ -381,6 +165,13 @@ public class SVNUpdater {
|
|||
}
|
||||
|
||||
|
||||
public void updateAlgorithmFilesProd(File a) throws SVNException{
|
||||
//this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a);
|
||||
this.updateAlgorithmList(this.configuration.getProdRepository(), a);
|
||||
}
|
||||
|
||||
|
||||
|
||||
public void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException {
|
||||
try {
|
||||
System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo);
|
||||
|
|
@ -533,6 +324,7 @@ public class SVNUpdater {
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException {
|
||||
//Timestamp timestamp = new Timestamp(System.currentTimeMillis());
|
||||
//long unixTime = System.currentTimeMillis() / 1000L;
|
||||
|
|
@ -547,6 +339,9 @@ public class SVNUpdater {
|
|||
+ algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
|
||||
sb.append("none | ");
|
||||
sb.append(this.getTimeZone() + " | ");
|
||||
|
||||
System.out.println("Algo details: "+sb.toString());
|
||||
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
|
|
@ -746,7 +541,6 @@ public class SVNUpdater {
|
|||
SVNFileUtil.closeFile(checksumStream);
|
||||
}
|
||||
}
|
||||
|
||||
public List<String> checkMatch(String[] lines, List<String> ls) {
|
||||
Set<String> ss = new HashSet<String>(ls);
|
||||
ss.addAll(Arrays.asList(lines));
|
||||
|
|
@ -758,7 +552,7 @@ public class SVNUpdater {
|
|||
Calendar cal = Calendar.getInstance();
|
||||
cal.getTime();
|
||||
DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy");
|
||||
Date fromDate = (Date)formatter.parse(cal.getTime().toString());
|
||||
Date fromDate = formatter.parse(cal.getTime().toString());
|
||||
TimeZone central = TimeZone.getTimeZone("UTC");
|
||||
formatter.setTimeZone(central);
|
||||
System.out.println(formatter.format(fromDate));
|
||||
|
|
@ -782,7 +576,7 @@ public class SVNUpdater {
|
|||
Calendar cal = Calendar.getInstance();
|
||||
cal.getTime();
|
||||
DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy");
|
||||
Date fromDate = (Date)formatter.parse(cal.getTime().toString());
|
||||
Date fromDate = formatter.parse(cal.getTime().toString());
|
||||
TimeZone central = TimeZone.getTimeZone("UTC");
|
||||
formatter.setTimeZone(central);
|
||||
System.out.println(formatter.format(fromDate));
|
||||
|
|
|
|||
|
|
@ -114,6 +114,9 @@ public class AlgorithmPackageParserTest {
|
|||
// "/home/ngalante/.proxy-settings"));
|
||||
|
||||
new AlgorithmPackageParserTest().extractAllAlgorithms();
|
||||
// AlgorithmPackageParserTest at = new AlgorithmPackageParserTest();
|
||||
// Algorithm a = at.extractAlgorithm("http://data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0");
|
||||
// System.out.println(a.getFullname());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue