This commit is contained in:
Nunzio Andrea Galante 2017-10-27 17:18:42 +00:00
parent 248e729e88
commit 46b04ae736
9 changed files with 356 additions and 327 deletions

View File

@ -8,4 +8,7 @@
<Changeset component="dataminer-pool-manager.2-1-0" date="${buildDate***REMOVED***">
<Change>New configuration file added</Change>
</Changeset>
<Changeset component="dataminer-pool-manager.2-1-1" date="${buildDate***REMOVED***">
<Change>Improvements and bugs fixed/SVN-UTF8 compliant/Installation in Production ghost added</Change>
</Changeset>
</ReleaseNotes>

View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<packaging>war</packaging>
<version>2.1.0-SNAPSHOT</version>
<version>2.1.1-SNAPSHOT</version>
<name>dataminer-pool-manager</name>
<description>

View File

@ -42,9 +42,9 @@ public class DataminerPoolManager {
public String publishAlgorithm(Algorithm algo, String targetVRE, String category, String algorithm_type/*, String env*/) throws IOException, InterruptedException {
***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
Cluster prodCluster = ClusterBuilder.getProdDataminerCluster();
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, /*prodCluster,*/ targetVRE, category, algorithm_type/*,env*/);
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/);
String id = job.start();
return id;
***REMOVED***

View File

@ -4,20 +4,23 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
import java.util.Collection;
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
public class ProductionPublishingJob extends DMPMJob {
private Algorithm algorithm;
***REMOVED***private Cluster prodCluster;
private Cluster prodCluster;
private String targetVREName;
private String category;
private String algorithm_type;
@ -26,41 +29,81 @@ public class ProductionPublishingJob extends DMPMJob {
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm,
/*Cluster prodCluster,*/ String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException {
Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException {
super(svnUpdater);
this.jobLogs = new File(
System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
this.jobLogs.mkdirs();
this.algorithm = algorithm;
***REMOVED***this.prodCluster = prodCluster;
this.prodCluster = prodCluster;
this.targetVREName = targetVREName;
this.category = category;
this.algorithm_type = algorithm_type;
***REMOVED***this.targetVREToken = targetVREToken;
***REMOVED***this.env= env;
this.jobLogs = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs");
this.jobLogs.mkdirs();
this.getStatus(0);
***REMOVED***
@Override
protected void execute() {
SendMail sm = new SendMail();
NotificationHelper nh = new NotificationHelper();
ServiceConfiguration a = new ServiceConfiguration();
CheckMethod b = new CheckMethod();
try {
***REMOVED***if (CheckPermission.apply(targetVREToken,targetVREName)){
***REMOVED***this.svnUpdater.updateProdDeps(this.algorithm);
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, this.category,this.algorithm_type, this.algorithm.getFullname()/*, env*/);
this.getStatus(9);
sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBodyRelease("\n\n"+this.buildInfo()));
Collection<String> undefinedDependencies = this.svnUpdater.getUndefinedDependencies(
this.svnUpdater.getDependencyFileProd(this.algorithm.getLanguage()/*,env*/),
this.algorithm.getDependencies());
if (!undefinedDependencies.isEmpty()) {
String message = "Following dependencies are not defined:\n";
for (String n : undefinedDependencies) {
message += "\n" + n +"\n";
***REMOVED***
this.getStatus(2);
sm.sendNotification(nh.getFailedSubjectRelease() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo()));
return;
***REMOVED******REMOVED***
***REMOVED***else this.getStatus(0);
***REMOVED***sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBodyRelease(" The user "+this.algorithm.getFullname()+ " is not authorized to access to the "+ targetVREName+ " VRE"+"\n\n"+this.buildInfo()));
***REMOVED***return;
***REMOVED*** int ret = this.executeAnsibleWorker(
***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube"));
***REMOVED***
b.deleteFilesProd(this.algorithm);
int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.prodCluster, false, "root"));
if (ret != 0) {
this.getStatus(2);
sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo());
return;
***REMOVED***
if (ret == 0) {
this.getStatus(0);
if (b.checkMethod(a.getProdHost(), SecurityTokenProvider.instance.get())&&(b.algoExistsProd(this.algorithm/*, env*/))) {
System.out.println("Interface check ok!");
System.out.println("Both the files exist at the correct path!");
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName,this.category, this.algorithm_type,
this.algorithm.getFullname()/*, env*/);
this.getStatus(9);
sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo()));
return;
***REMOVED*** else
this.getStatus(2);
sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm",
nh.getFailedBody(
"\n"+
"Installation completed but DataMiner Interface not working correctly or files "
+ this.algorithm.getName() + ".jar and " + this.algorithm.getName()
+ "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo());
return;
***REMOVED***
***REMOVED*** catch (Exception e) {
try {
this.getStatus(0);
@ -75,42 +118,51 @@ public class ProductionPublishingJob extends DMPMJob {
***REMOVED***
***REMOVED***
public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException {
File m = new File(this.jobLogs + File.separator + this.id+"_exitStatus");
File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus");
PrintWriter writer = new PrintWriter(m, "UTF-8");
String response = "";
if (exitstatus == 0) {
response = "IN PROGRESS";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
if (exitstatus == 9) {
response = "COMPLETED";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
if (exitstatus == 0) {
if (exitstatus == 2) {
response = "FAILED";
writer.println(response);
***REMOVED***writer.close();
***REMOVED***
writer.close();
return response;
***REMOVED***
public String buildInfo(){
public String buildInfo() throws UnsupportedEncodingException{
ServiceConfiguration a = new ServiceConfiguration();
return
"\n"+
"Algorithm details:\n"+"\n"+
"User: "+this.algorithm.getFullname()+"\n"+
"Algorithm name: "+this.algorithm.getName()+"\n"+
"Staging DataMiner Host: "+ a.getProdHost()+"\n"+
"Caller VRE: "+ScopeProvider.instance.get()+"\n"+
"Target VRE: "+targetVREName+"\n";
"Target VRE: "+this.targetVREName+"\n";
***REMOVED***
***REMOVED***

View File

@ -87,7 +87,6 @@ public class CheckMethod {
***REMOVED***
public boolean algoExists(Algorithm a/*, String env*/) throws Exception{
ServiceConfiguration p = new ServiceConfiguration();
@ -112,6 +111,30 @@ public class CheckMethod {
***REMOVED***
public boolean algoExistsProd(Algorithm a/*, String env*/) throws Exception{
ServiceConfiguration p = new ServiceConfiguration();
File file = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
if ((this.doesExistProd(file.getPath()/*,env*/)) && (this.doesExistProd(file2.getPath()/*,env*/))){
this.copyFromDmToSVNProd(file/*,env*/);
this.copyFromDmToSVNProd(file2/*,env*/);
return true;
***REMOVED***
else
System.out.println("Algorithm "+a.getName()+".jar"+ " and "+a.getName()+"_interface.jar files are not present at the expected path");
return false;
***REMOVED***
public void deleteFiles(Algorithm a/*,String env*/) throws Exception{
JSch jsch = new JSch();
Session session = null;
@ -163,7 +186,55 @@ public class CheckMethod {
***REMOVED***
public void deleteFilesProd(Algorithm a/*,String env*/) throws Exception{
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
ServiceConfiguration p = new ServiceConfiguration();
System.out.println("checking existing in env: " + p.getProdHost());
File file = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+".jar");
File file2 = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar");
System.out.println("First file is located to: "+file.getPath());
System.out.println("Second file is located to: "+file2.getPath());
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", p.getProdHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
if(doesExistProd(file.getPath()/*,env*/)&&(doesExistProd(file2.getPath()/*,env*/))){
c.rm(file.getPath());
c.rm(file2.getPath());
System.out.println("Both the files have been deleted");
***REMOVED***
else System.out.println("Files not found");
channel.disconnect();
c.disconnect();
session.disconnect();
***REMOVED***
@ -219,6 +290,55 @@ public class CheckMethod {
public boolean doesExistProd(String path/*, String env*/) throws Exception {
JSch jsch = new JSch();
Session session = null;
Channel channel = null;
ChannelSftp c = null;
boolean success = false;
ServiceConfiguration p = new ServiceConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", p.getProdHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
channel = session.openChannel("sftp");
channel.connect();
System.out.println("shell channel connected....");
c = (ChannelSftp) channel;
SftpATTRS is = null;
System.out.println(path);
try {
is = c.lstat(path);
success = true;
***REMOVED*** catch (SftpException e) {
if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
***REMOVED*** file doesn't exist
success = false;
***REMOVED***
***REMOVED***success = true; ***REMOVED*** something else went wrong
***REMOVED***
channel.disconnect();
c.disconnect();
session.disconnect();
return success;
***REMOVED***
public void copyFromDmToSVN(File a/*,String env*/) throws Exception {
@ -264,9 +384,50 @@ public class CheckMethod {
f.delete();
***REMOVED***
public void copyFromDmToSVNProd(File a/*,String env*/) throws Exception {
JSch jsch = new JSch();
Session session = null;
ServiceConfiguration sc = new ServiceConfiguration();
SVNUpdater svnUpdater = new SVNUpdater(sc);
ServiceConfiguration p = new ServiceConfiguration();
jsch.setKnownHosts("~/.ssh/known_hosts");
String privateKey = "~/.ssh/id_rsa";
jsch.addIdentity(privateKey);
System.out.println("Private Key Added.");
session = jsch.getSession("root", p.getProdHost());
System.out.println("session created.");
java.util.Properties config = new java.util.Properties();
config.put("StrictHostKeyChecking", "no");
session.setConfig(config);
session.connect();
Channel channel = session.openChannel("sftp");
channel.connect();
ChannelSftp sftp = (ChannelSftp) channel;
sftp.cd(p.getProdGhostAlgoDirectory());
System.out.println("REMOTE : "+p.getProdGhostAlgoDirectory()+"/"+a.getName());
System.out.println("LOCAL : /tmp/"+a.getName());
sftp.get(p.getProdGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName());
channel.disconnect();
session.disconnect();
File f = new File("/tmp/"+a.getName());
svnUpdater.updateAlgorithmFilesProd(f);
f.delete();
***REMOVED***
public List<String> getFiles(String a){
@ -281,13 +442,6 @@ public class CheckMethod {
return ls;
***REMOVED***
public static void main(String[] args) throws Exception {
***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
@ -295,7 +449,7 @@ public class CheckMethod {
CheckMethod a = new CheckMethod();
a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt");
***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar");
***REMOVED*** System.out.println(aa.getName());
@ -321,8 +475,8 @@ public class CheckMethod {
***REMOVED*** a.copyFromDmToSVN(aa, "Dev");
***REMOVED***a.checkMethod("dataminer-ghost-d.dev.d4science.org",
***REMOVED*** "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548");
System.out.println(a.checkMethod("dm-pool-manager-proto.d4science.org",
"2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548"));

View File

@ -33,6 +33,29 @@ public class ClusterBuilder {
return cluster;
***REMOVED***
public static Cluster getProdDataminerCluster() throws FileNotFoundException{
Cluster cluster = new Cluster();
ServiceConfiguration p = new ServiceConfiguration();
Host h = new Host();
h.setName(p.getProdHost());
cluster.addHost(h);
***REMOVED*** if (env.equals("Dev")){
***REMOVED*** h.setName(p.getDevStagingHost());
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto")))){
***REMOVED*** h.setName(p.getProtoProdStagingHost());
***REMOVED*** cluster.addHost(h);
***REMOVED*** ***REMOVED***
return cluster;
***REMOVED***
public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException{
Cluster cluster = new Cluster();
for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) {

View File

@ -15,11 +15,11 @@ public class NotificationHelper {
***REMOVED***
private String getSubjectHeader() {
return "[DataMinerGhostInstallationRequestReport]";
return "[DataMinerGhostStagingInstallationRequestReport]";
***REMOVED***
private String getSubjectHeaderRelease() {
return "[DataMinerReleaseInstallationRequestReport]";
return "[DataMinerGhostProductionInstallationRequestReport]";
***REMOVED***
private boolean isError() {
@ -49,7 +49,7 @@ public class NotificationHelper {
***REMOVED***
public String getSuccessBody(String info) {
String message = String.format("The installation of the algorithm in the ghost dataminer is completed successfully.");
String message = String.format("The installation of the algorithm is completed successfully.");
message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info;
return message;
***REMOVED***
@ -60,17 +60,17 @@ public class NotificationHelper {
return body;
***REMOVED***
public String getSuccessBodyRelease(String info) {
String message = String.format("SVN REPOSITORY CORRECTLY UPDATED.");
message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info;
return message;
***REMOVED***
public String getFailedBodyRelease(String info) {
String message = String.format("SVN REPOSITORY UPDATE FAILED.");
message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info;
return message;
***REMOVED***
***REMOVED*** public String getSuccessBodyRelease(String info) {
***REMOVED*** String message = String.format("SVN REPOSITORY CORRECTLY UPDATED.");
***REMOVED*** message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info;
***REMOVED*** return message;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** public String getFailedBodyRelease(String info) {
***REMOVED*** String message = String.format("SVN REPOSITORY UPDATE FAILED.");
***REMOVED*** message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info;
***REMOVED*** return message;
***REMOVED*** ***REMOVED***
***REMOVED*** public String getSubject() {
***REMOVED*** if(this.isError()) {

View File

@ -19,14 +19,12 @@ import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
***REMOVED***
import java.sql.Timestamp;
import java.io.UnsupportedEncodingException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
import javax.ws.rs.core.NewCookie;
/**
* Created by ggiammat on 5/9/17.
*/
@ -72,116 +70,32 @@ public class SVNUpdater {
return a;
***REMOVED***
public String getDependencyFileProd(String language/*, String env*/){
String a = "";
***REMOVED*** if (env.equals("Dev")){
***REMOVED*** a= this.getDevDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Prod")){
***REMOVED*** a= this.getProdDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Proto")){
***REMOVED*** a= this.getRProtoDependencyFile(language);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Preprod")){
***REMOVED*** a= this.getPreProdDependencyFile(language);
***REMOVED*** ***REMOVED***
a = this.getProdDependencyFile(language);
return a;
***REMOVED***
***REMOVED*** public String getRProtoDependencyFile(String language) {
***REMOVED*** switch (language) {
***REMOVED*** case "R":
***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList();
***REMOVED*** case "R-blackbox":
***REMOVED*** return this.configuration.getSVNRProtoRBDepsList();
***REMOVED*** case "Java":
***REMOVED*** return this.configuration.getSVNRProtoJavaDepsList();
***REMOVED*** case "Knime-Workflow":
***REMOVED*** return this.configuration.getSVNRProtoKWDepsList();
***REMOVED*** case "Linux-compiled":
***REMOVED*** return this.configuration.getSVNRProtoLinuxCompiledDepsList();
***REMOVED*** case "Octave":
***REMOVED*** return this.configuration.getSVNRProtoOctaveDepsList();
***REMOVED*** case "Python":
***REMOVED*** return this.configuration.getSVNRProtoPythonDepsList();
***REMOVED*** case "Pre-Installed":
***REMOVED*** return this.configuration.getSVNRProtoPreInstalledDepsList();
***REMOVED*** case "Windows-compiled":
***REMOVED*** return this.configuration.getSVNRProtoWCDepsList();
***REMOVED*** default:
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** public String getPreProdDependencyFile(String language) {
***REMOVED*** switch (language) {
***REMOVED*** case "R":
***REMOVED*** return this.configuration.getSVNPreProdCRANDepsList();
***REMOVED*** case "R-blackbox":
***REMOVED*** return this.configuration.getSVNPreProdRBDepsList();
***REMOVED*** case "Java":
***REMOVED*** return this.configuration.getSVNPreProdJavaDepsList();
***REMOVED*** case "Knime-Workflow":
***REMOVED*** return this.configuration.getSVNPreProdKWDepsList();
***REMOVED*** case "Linux-compiled":
***REMOVED*** return this.configuration.getSVNPreProdLinuxCompiledDepsList();
***REMOVED*** case "Octave":
***REMOVED*** return this.configuration.getSVNPreProdOctaveDepsList();
***REMOVED*** case "Python":
***REMOVED*** return this.configuration.getSVNPreProdPythonDepsList();
***REMOVED*** case "Pre-Installed":
***REMOVED*** return this.configuration.getSVNPreProdPreInstalledDepsList();
***REMOVED*** case "Windows-compiled":
***REMOVED*** return this.configuration.getSVNPreProdWCDepsList();
***REMOVED*** default:
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** public String getProdDependencyFile(String language) {
***REMOVED*** switch (language) {
***REMOVED*** case "R":
***REMOVED*** return this.configuration.getSVNProdCRANDepsList();
***REMOVED*** case "R-blackbox":
***REMOVED*** return this.configuration.getSVNProdRBDepsList();
***REMOVED*** case "Java":
***REMOVED*** return this.configuration.getSVNProdJavaDepsList();
***REMOVED*** case "Knime-Workflow":
***REMOVED*** return this.configuration.getSVNProdKWDepsList();
***REMOVED*** case "Linux-compiled":
***REMOVED*** return this.configuration.getSVNProdLinuxCompiledDepsList();
***REMOVED*** case "Octave":
***REMOVED*** return this.configuration.getSVNProdOctaveDepsList();
***REMOVED*** case "Python":
***REMOVED*** return this.configuration.getSVNProdPythonDepsList();
***REMOVED*** case "Pre-Installed":
***REMOVED*** return this.configuration.getSVNProdPreInstalledDepsList();
***REMOVED*** case "Windows-compiled":
***REMOVED*** return this.configuration.getSVNProdWCDepsList();
***REMOVED*** default:
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** public String getDevDependencyFile(String language) {
***REMOVED*** switch (language) {
***REMOVED*** case "R":
***REMOVED*** return this.configuration.getSVNDevCRANDepsList();
***REMOVED*** case "R-blackbox":
***REMOVED*** return this.configuration.getSVNDevRBDepsList();
***REMOVED*** case "Java":
***REMOVED*** return this.configuration.getSVNDevJavaDepsList();
***REMOVED*** case "Knime-Workflow":
***REMOVED*** return this.configuration.getSVNDevKWDepsList();
***REMOVED*** case "Linux-compiled":
***REMOVED*** return this.configuration.getSVNDevLinuxCompiledDepsList();
***REMOVED*** case "Octave":
***REMOVED*** return this.configuration.getSVNDevOctaveDepsList();
***REMOVED*** case "Python":
***REMOVED*** return this.configuration.getSVNDevPythonDepsList();
***REMOVED*** case "Pre-Installed":
***REMOVED*** return this.configuration.getSVNDevPreInstalledDepsList();
***REMOVED*** case "Windows-compiled":
***REMOVED*** return this.configuration.getSVNDevWCDepsList();
***REMOVED*** default:
***REMOVED*** return null;
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
public String getStagingDependencyFile(String language) {
switch (language) {
@ -209,162 +123,32 @@ public class SVNUpdater {
***REMOVED***
***REMOVED*** public void readPreProdDeps(Algorithm algorithm) throws SVNException {
***REMOVED*** if (algorithm.getLanguage().equals("R")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Java")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Octave")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Python")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
public String getProdDependencyFile(String language) {
switch (language) {
case "R":
return this.configuration.getSVNProdCRANDepsList();
case "R-blackbox":
return this.configuration.getSVNProdRBDepsList();
case "Java":
return this.configuration.getSVNProdJavaDepsList();
case "Knime-Workflow":
return this.configuration.getSVNProdKWDepsList();
case "Linux-compiled":
return this.configuration.getSVNProdLinuxCompiledDepsList();
case "Octave":
return this.configuration.getSVNProdOctaveDepsList();
case "Python":
return this.configuration.getSVNProdPythonDepsList();
case "Pre-Installed":
return this.configuration.getSVNProdPreInstalledDepsList();
case "Windows-compiled":
return this.configuration.getSVNProdWCDepsList();
default:
return null;
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public void readRPRotoDeps(Algorithm algorithm) throws SVNException {
***REMOVED*** if (algorithm.getLanguage().equals("R")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Java")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Octave")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Python")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED*** public void readProdDeps(Algorithm algorithm) throws SVNException {
***REMOVED*** if (algorithm.getLanguage().equals("R")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNProdCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Java")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Octave")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Python")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** public void readRDevDeps(Algorithm algorithm) throws SVNException {
***REMOVED*** if (algorithm.getLanguage().equals("R")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Java")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) {
***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Octave")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Python")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) {
***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies());
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** public void updateProdDeps(Algorithm algorithm) {
***REMOVED*** this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
***REMOVED*** this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
***REMOVED******REMOVED***
***REMOVED*** public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
***REMOVED***
***REMOVED*** if (env.equals("Dev")){
***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNDevAlgorithmsList(), algorithm, targetVRE, category,algorithm_type, user, env);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Prod")){
***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Proto")){
***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** if (env.equals("Preprod")){
***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNPreProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env);
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
public void updateSVNStagingAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) {
this.updateSVNAlgorithmList(this.configuration.getSVNStagingAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user);
@ -381,6 +165,13 @@ public class SVNUpdater {
***REMOVED***
public void updateAlgorithmFilesProd(File a) throws SVNException{
***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a);
this.updateAlgorithmList(this.configuration.getProdRepository(), a);
***REMOVED***
public void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException {
try {
System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo);
@ -533,6 +324,7 @@ public class SVNUpdater {
***REMOVED***
***REMOVED***
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException {
***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis());
***REMOVED***long unixTime = System.currentTimeMillis() / 1000L;
@ -547,6 +339,9 @@ public class SVNUpdater {
+ algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
sb.append("none | ");
sb.append(this.getTimeZone() + " | ");
System.out.println("Algo details: "+sb.toString());
return sb.toString();
***REMOVED***
@ -746,7 +541,6 @@ public class SVNUpdater {
SVNFileUtil.closeFile(checksumStream);
***REMOVED***
***REMOVED***
public List<String> checkMatch(String[] lines, List<String> ls) {
Set<String> ss = new HashSet<String>(ls);
ss.addAll(Arrays.asList(lines));
@ -758,7 +552,7 @@ public class SVNUpdater {
Calendar cal = Calendar.getInstance();
cal.getTime();
DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy");
Date fromDate = (Date)formatter.parse(cal.getTime().toString());
Date fromDate = formatter.parse(cal.getTime().toString());
TimeZone central = TimeZone.getTimeZone("UTC");
formatter.setTimeZone(central);
System.out.println(formatter.format(fromDate));
@ -782,7 +576,7 @@ public class SVNUpdater {
Calendar cal = Calendar.getInstance();
cal.getTime();
DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy");
Date fromDate = (Date)formatter.parse(cal.getTime().toString());
Date fromDate = formatter.parse(cal.getTime().toString());
TimeZone central = TimeZone.getTimeZone("UTC");
formatter.setTimeZone(central);
System.out.println(formatter.format(fromDate));

View File

@ -114,6 +114,9 @@ public class AlgorithmPackageParserTest {
***REMOVED*** "/home/ngalante/.proxy-settings"));
new AlgorithmPackageParserTest().extractAllAlgorithms();
***REMOVED*** AlgorithmPackageParserTest at = new AlgorithmPackageParserTest();
***REMOVED*** Algorithm a = at.extractAlgorithm("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0");
***REMOVED*** System.out.println(a.getFullname());
***REMOVED***
***REMOVED***