diff --git a/distro/changelog.xml b/distro/changelog.xml index 11af390..c4da3fe 100644 --- a/distro/changelog.xml +++ b/distro/changelog.xml @@ -8,4 +8,7 @@ New configuration file added + + Improvements and bugs fixed/SVN-UTF8 compliant/Installation in Production ghost added + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5141838..858f99d 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ org.gcube.dataAnalysis dataminer-pool-manager war - 2.1.0-SNAPSHOT + 2.1.1-SNAPSHOT dataminer-pool-manager diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 16dbb1b..784192a 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -42,9 +42,9 @@ public class DataminerPoolManager { public String publishAlgorithm(Algorithm algo, String targetVRE, String category, String algorithm_type/*, String env*/) throws IOException, InterruptedException { - ***REMOVED***Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); + Cluster prodCluster = ClusterBuilder.getProdDataminerCluster(); - DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, /*prodCluster,*/ targetVRE, category, algorithm_type/*,env*/); + DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE, category, algorithm_type/*,env*/); String id = job.start(); return id; ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java index a0c954c..898dde8 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -4,20 +4,23 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.PrintWriter; import java.io.UnsupportedEncodingException; +import java.util.Collection; ***REMOVED*** ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; ***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.CheckMethod; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.util.NotificationHelper; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SendMail; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration; public class ProductionPublishingJob extends DMPMJob { private Algorithm algorithm; - ***REMOVED***private Cluster prodCluster; + private Cluster prodCluster; private String targetVREName; private String category; private String algorithm_type; @@ -26,41 +29,81 @@ public class ProductionPublishingJob extends DMPMJob { public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, - /*Cluster prodCluster,*/ String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException { + Cluster prodCluster, String targetVREName, String category,String algorithm_type/*, String env*/) throws FileNotFoundException, UnsupportedEncodingException { super(svnUpdater); + this.jobLogs = new File( + System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs"); + this.jobLogs.mkdirs(); this.algorithm = algorithm; - ***REMOVED***this.prodCluster = prodCluster; + this.prodCluster = prodCluster; this.targetVREName = targetVREName; this.category = category; this.algorithm_type = algorithm_type; ***REMOVED***this.targetVREToken = targetVREToken; ***REMOVED***this.env= env; - - - this.jobLogs = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager" + File.separator + "jobs"); - this.jobLogs.mkdirs(); + this.getStatus(0); + ***REMOVED*** @Override protected void execute() { SendMail sm = new SendMail(); NotificationHelper nh = new NotificationHelper(); + ServiceConfiguration a = new ServiceConfiguration(); + CheckMethod b = new CheckMethod(); try { - - ***REMOVED***if (CheckPermission.apply(targetVREToken,targetVREName)){ - - ***REMOVED***this.svnUpdater.updateProdDeps(this.algorithm); - this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, this.category,this.algorithm_type, this.algorithm.getFullname()/*, env*/); - this.getStatus(9); - sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBodyRelease("\n\n"+this.buildInfo())); + + Collection undefinedDependencies = this.svnUpdater.getUndefinedDependencies( + this.svnUpdater.getDependencyFileProd(this.algorithm.getLanguage()/*,env*/), + this.algorithm.getDependencies()); + + if (!undefinedDependencies.isEmpty()) { + + String message = "Following dependencies are not defined:\n"; + for (String n : undefinedDependencies) { + message += "\n" + n +"\n"; + ***REMOVED*** + this.getStatus(2); + + sm.sendNotification(nh.getFailedSubjectRelease() +" for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody(message+"\n\n"+this.buildInfo())); return; - ***REMOVED******REMOVED*** - ***REMOVED***else this.getStatus(0); - ***REMOVED***sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBodyRelease(" The user "+this.algorithm.getFullname()+ " is not authorized to access to the "+ targetVREName+ " VRE"+"\n\n"+this.buildInfo())); - ***REMOVED***return; -***REMOVED*** int ret = this.executeAnsibleWorker( -***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube")); - + ***REMOVED*** + b.deleteFilesProd(this.algorithm); + + int ret = this.executeAnsibleWorker(createWorker(this.algorithm, this.prodCluster, false, "root")); + + if (ret != 0) { + this.getStatus(2); + sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getFailedBody("Installation failed. Return code=" + ret)+"\n\n"+this.buildInfo()); + return; + ***REMOVED*** + + + if (ret == 0) { + this.getStatus(0); + + if (b.checkMethod(a.getProdHost(), SecurityTokenProvider.instance.get())&&(b.algoExistsProd(this.algorithm/*, env*/))) { + + System.out.println("Interface check ok!"); + System.out.println("Both the files exist at the correct path!"); + + this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName,this.category, this.algorithm_type, + this.algorithm.getFullname()/*, env*/); + + this.getStatus(9); + sm.sendNotification(nh.getSuccessSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", nh.getSuccessBody("\n\n"+this.buildInfo())); + return; + ***REMOVED*** else + this.getStatus(2); + sm.sendNotification(nh.getFailedSubjectRelease() + " for "+this.algorithm.getName()+ " algorithm", + nh.getFailedBody( + "\n"+ + "Installation completed but DataMiner Interface not working correctly or files " + + this.algorithm.getName() + ".jar and " + this.algorithm.getName() + + "_interface.jar not availables at the expected path")+"\n\n"+this.buildInfo()); + return; + + ***REMOVED*** ***REMOVED*** catch (Exception e) { try { this.getStatus(0); @@ -75,42 +118,51 @@ public class ProductionPublishingJob extends DMPMJob { ***REMOVED*** ***REMOVED*** + + public String getStatus(int exitstatus) throws FileNotFoundException, UnsupportedEncodingException { - File m = new File(this.jobLogs + File.separator + this.id+"_exitStatus"); + File m = new File(this.jobLogs + File.separator + this.id + "_exitStatus"); PrintWriter writer = new PrintWriter(m, "UTF-8"); String response = ""; + if (exitstatus == 0) { + response = "IN PROGRESS"; + writer.println(response); + ***REMOVED***writer.close(); + ***REMOVED*** + if (exitstatus == 9) { response = "COMPLETED"; writer.println(response); + ***REMOVED***writer.close(); ***REMOVED*** - - if (exitstatus == 0) { + + if (exitstatus == 2) { response = "FAILED"; writer.println(response); ***REMOVED***writer.close(); ***REMOVED*** - - writer.close(); return response; ***REMOVED*** + - -public String buildInfo(){ + public String buildInfo() throws UnsupportedEncodingException{ + ServiceConfiguration a = new ServiceConfiguration(); + return "\n"+ "Algorithm details:\n"+"\n"+ "User: "+this.algorithm.getFullname()+"\n"+ "Algorithm name: "+this.algorithm.getName()+"\n"+ + "Staging DataMiner Host: "+ a.getProdHost()+"\n"+ "Caller VRE: "+ScopeProvider.instance.get()+"\n"+ - "Target VRE: "+targetVREName+"\n"; + "Target VRE: "+this.targetVREName+"\n"; ***REMOVED*** - ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java index 9059ca3..ac63dc0 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/CheckMethod.java @@ -87,7 +87,6 @@ public class CheckMethod { ***REMOVED*** - public boolean algoExists(Algorithm a/*, String env*/) throws Exception{ ServiceConfiguration p = new ServiceConfiguration(); @@ -112,6 +111,30 @@ public class CheckMethod { ***REMOVED*** + public boolean algoExistsProd(Algorithm a/*, String env*/) throws Exception{ + ServiceConfiguration p = new ServiceConfiguration(); + + File file = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+".jar"); + File file2 = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); + + + System.out.println("First file is located to: "+file.getPath()); + System.out.println("Second file is located to: "+file2.getPath()); + + + if ((this.doesExistProd(file.getPath()/*,env*/)) && (this.doesExistProd(file2.getPath()/*,env*/))){ + this.copyFromDmToSVNProd(file/*,env*/); + this.copyFromDmToSVNProd(file2/*,env*/); + + + return true; + ***REMOVED*** + else + System.out.println("Algorithm "+a.getName()+".jar"+ " and "+a.getName()+"_interface.jar files are not present at the expected path"); + return false; +***REMOVED*** + + public void deleteFiles(Algorithm a/*,String env*/) throws Exception{ JSch jsch = new JSch(); Session session = null; @@ -163,7 +186,55 @@ public class CheckMethod { ***REMOVED*** - + public void deleteFilesProd(Algorithm a/*,String env*/) throws Exception{ + JSch jsch = new JSch(); + Session session = null; + Channel channel = null; + ChannelSftp c = null; + ServiceConfiguration p = new ServiceConfiguration(); + System.out.println("checking existing in env: " + p.getProdHost()); + + File file = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+".jar"); + File file2 = new File(p.getProdGhostAlgoDirectory()+"/"+a.getName()+"_interface.jar"); + + + System.out.println("First file is located to: "+file.getPath()); + System.out.println("Second file is located to: "+file2.getPath()); + + + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + session = jsch.getSession("root", p.getProdHost()); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + channel = session.openChannel("sftp"); + channel.connect(); + System.out.println("shell channel connected...."); + + c = (ChannelSftp) channel; + + if(doesExistProd(file.getPath()/*,env*/)&&(doesExistProd(file2.getPath()/*,env*/))){ + + c.rm(file.getPath()); + c.rm(file2.getPath()); + System.out.println("Both the files have been deleted"); + ***REMOVED*** + else System.out.println("Files not found"); + channel.disconnect(); + c.disconnect(); + session.disconnect(); + +***REMOVED*** @@ -219,6 +290,55 @@ public class CheckMethod { + public boolean doesExistProd(String path/*, String env*/) throws Exception { + JSch jsch = new JSch(); + Session session = null; + Channel channel = null; + ChannelSftp c = null; + boolean success = false; + ServiceConfiguration p = new ServiceConfiguration(); + + + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + session = jsch.getSession("root", p.getProdHost()); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + channel = session.openChannel("sftp"); + channel.connect(); + System.out.println("shell channel connected...."); + + c = (ChannelSftp) channel; + + SftpATTRS is = null; + System.out.println(path); + + try { + is = c.lstat(path); + success = true; + ***REMOVED*** catch (SftpException e) { + if (e.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) { + ***REMOVED*** file doesn't exist + success = false; + ***REMOVED*** + ***REMOVED***success = true; ***REMOVED*** something else went wrong + ***REMOVED*** + channel.disconnect(); + c.disconnect(); + session.disconnect(); + return success; + +***REMOVED*** public void copyFromDmToSVN(File a/*,String env*/) throws Exception { @@ -264,9 +384,50 @@ public class CheckMethod { f.delete(); ***REMOVED*** + + public void copyFromDmToSVNProd(File a/*,String env*/) throws Exception { + JSch jsch = new JSch(); + Session session = null; + ServiceConfiguration sc = new ServiceConfiguration(); + SVNUpdater svnUpdater = new SVNUpdater(sc); + + ServiceConfiguration p = new ServiceConfiguration(); + + jsch.setKnownHosts("~/.ssh/known_hosts"); + String privateKey = "~/.ssh/id_rsa"; + + jsch.addIdentity(privateKey); + System.out.println("Private Key Added."); + + session = jsch.getSession("root", p.getProdHost()); + System.out.println("session created."); + + java.util.Properties config = new java.util.Properties(); + config.put("StrictHostKeyChecking", "no"); + + session.setConfig(config); + session.connect(); + + Channel channel = session.openChannel("sftp"); + channel.connect(); + + ChannelSftp sftp = (ChannelSftp) channel; + + sftp.cd(p.getProdGhostAlgoDirectory()); + + System.out.println("REMOTE : "+p.getProdGhostAlgoDirectory()+"/"+a.getName()); + System.out.println("LOCAL : /tmp/"+a.getName()); + + sftp.get(p.getProdGhostAlgoDirectory()+"/"+a.getName(),"/tmp/"+a.getName()); + + channel.disconnect(); + session.disconnect(); - + File f = new File("/tmp/"+a.getName()); + svnUpdater.updateAlgorithmFilesProd(f); + f.delete(); +***REMOVED*** public List getFiles(String a){ @@ -281,13 +442,6 @@ public class CheckMethod { return ls; ***REMOVED*** - - - - - - - public static void main(String[] args) throws Exception { ***REMOVED*** ServiceConfiguration a = new ServiceConfiguration(); @@ -295,7 +449,7 @@ public class CheckMethod { CheckMethod a = new CheckMethod(); - a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); + ***REMOVED***a.getFiles("/trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt, /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt"); ***REMOVED*** File aa = new File("OCTAVEBLACKBOX.jar"); ***REMOVED*** System.out.println(aa.getName()); @@ -321,8 +475,8 @@ public class CheckMethod { ***REMOVED*** a.copyFromDmToSVN(aa, "Dev"); -***REMOVED***a.checkMethod("dataminer-ghost-d.dev.d4science.org", -***REMOVED*** "708e7eb8-11a7-4e9a-816b-c9ed7e7e99fe-98187548"); +System.out.println(a.checkMethod("dm-pool-manager-proto.d4science.org", + "2eceaf27-0e22-4dbe-8075-e09eff199bf9-98187548")); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java index 1a797d1..a790cb9 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java @@ -33,6 +33,29 @@ public class ClusterBuilder { return cluster; ***REMOVED*** + + public static Cluster getProdDataminerCluster() throws FileNotFoundException{ + Cluster cluster = new Cluster(); + ServiceConfiguration p = new ServiceConfiguration(); + Host h = new Host(); + h.setName(p.getProdHost()); + cluster.addHost(h); + +***REMOVED*** if (env.equals("Dev")){ +***REMOVED*** h.setName(p.getDevStagingHost()); +***REMOVED*** cluster.addHost(h); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** if ((env.equals("Prod")||(env.equals("Proto")))){ +***REMOVED*** h.setName(p.getProtoProdStagingHost()); +***REMOVED*** cluster.addHost(h); +***REMOVED*** ***REMOVED*** + + return cluster; +***REMOVED*** + + + public static Cluster getVRECluster(String targetVREToken, String targetVRE) throws IOException{ Cluster cluster = new Cluster(); for (Host h : new HAProxy().listDataMinersByCluster(targetVREToken,targetVRE)) { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java index 0f02bed..d64e86d 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/NotificationHelper.java @@ -15,11 +15,11 @@ public class NotificationHelper { ***REMOVED*** private String getSubjectHeader() { - return "[DataMinerGhostInstallationRequestReport]"; + return "[DataMinerGhostStagingInstallationRequestReport]"; ***REMOVED*** private String getSubjectHeaderRelease() { - return "[DataMinerReleaseInstallationRequestReport]"; + return "[DataMinerGhostProductionInstallationRequestReport]"; ***REMOVED*** private boolean isError() { @@ -49,7 +49,7 @@ public class NotificationHelper { ***REMOVED*** public String getSuccessBody(String info) { - String message = String.format("The installation of the algorithm in the ghost dataminer is completed successfully."); + String message = String.format("The installation of the algorithm is completed successfully."); message+="\n\nYou can retrieve experiment results under the '/DataMiner' e-Infrastructure Workspace folder or from the DataMiner interface.\n\n"+ info; return message; ***REMOVED*** @@ -60,17 +60,17 @@ public class NotificationHelper { return body; ***REMOVED*** - public String getSuccessBodyRelease(String info) { - String message = String.format("SVN REPOSITORY CORRECTLY UPDATED."); - message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info; - return message; - ***REMOVED*** - - public String getFailedBodyRelease(String info) { - String message = String.format("SVN REPOSITORY UPDATE FAILED."); - message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info; - return message; - ***REMOVED*** +***REMOVED*** public String getSuccessBodyRelease(String info) { +***REMOVED*** String message = String.format("SVN REPOSITORY CORRECTLY UPDATED."); +***REMOVED*** message+="\n\n The CRON job will install the algorithm in the target VRE \n\n"+ info; +***REMOVED*** return message; +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** public String getFailedBodyRelease(String info) { +***REMOVED*** String message = String.format("SVN REPOSITORY UPDATE FAILED."); +***REMOVED*** message+="\n\n The CRON job will NOT be able to install the algorithm in the target VRE \n\n"+ info; +***REMOVED*** return message; +***REMOVED*** ***REMOVED*** ***REMOVED*** public String getSubject() { ***REMOVED*** if(this.isError()) { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index b14d19d..7be2055 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -19,14 +19,12 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; ***REMOVED*** -import java.sql.Timestamp; +import java.io.UnsupportedEncodingException; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; -import javax.ws.rs.core.NewCookie; - /** * Created by ggiammat on 5/9/17. */ @@ -72,116 +70,32 @@ public class SVNUpdater { return a; ***REMOVED*** + + + +public String getDependencyFileProd(String language/*, String env*/){ + + String a = ""; +***REMOVED*** if (env.equals("Dev")){ +***REMOVED*** a= this.getDevDependencyFile(language); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** if (env.equals("Prod")){ +***REMOVED*** a= this.getProdDependencyFile(language); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** if (env.equals("Proto")){ +***REMOVED*** a= this.getRProtoDependencyFile(language); +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** if (env.equals("Preprod")){ +***REMOVED*** a= this.getPreProdDependencyFile(language); +***REMOVED*** ***REMOVED*** + a = this.getProdDependencyFile(language); + return a; +***REMOVED*** -***REMOVED*** public String getRProtoDependencyFile(String language) { -***REMOVED*** switch (language) { -***REMOVED*** case "R": -***REMOVED*** return this.configuration.getSVNRProtoCRANDepsList(); -***REMOVED*** case "R-blackbox": -***REMOVED*** return this.configuration.getSVNRProtoRBDepsList(); -***REMOVED*** case "Java": -***REMOVED*** return this.configuration.getSVNRProtoJavaDepsList(); -***REMOVED*** case "Knime-Workflow": -***REMOVED*** return this.configuration.getSVNRProtoKWDepsList(); -***REMOVED*** case "Linux-compiled": -***REMOVED*** return this.configuration.getSVNRProtoLinuxCompiledDepsList(); -***REMOVED*** case "Octave": -***REMOVED*** return this.configuration.getSVNRProtoOctaveDepsList(); -***REMOVED*** case "Python": -***REMOVED*** return this.configuration.getSVNRProtoPythonDepsList(); -***REMOVED*** case "Pre-Installed": -***REMOVED*** return this.configuration.getSVNRProtoPreInstalledDepsList(); -***REMOVED*** case "Windows-compiled": -***REMOVED*** return this.configuration.getSVNRProtoWCDepsList(); -***REMOVED*** default: -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** - - - -***REMOVED*** public String getPreProdDependencyFile(String language) { -***REMOVED*** switch (language) { -***REMOVED*** case "R": -***REMOVED*** return this.configuration.getSVNPreProdCRANDepsList(); -***REMOVED*** case "R-blackbox": -***REMOVED*** return this.configuration.getSVNPreProdRBDepsList(); -***REMOVED*** case "Java": -***REMOVED*** return this.configuration.getSVNPreProdJavaDepsList(); -***REMOVED*** case "Knime-Workflow": -***REMOVED*** return this.configuration.getSVNPreProdKWDepsList(); -***REMOVED*** case "Linux-compiled": -***REMOVED*** return this.configuration.getSVNPreProdLinuxCompiledDepsList(); -***REMOVED*** case "Octave": -***REMOVED*** return this.configuration.getSVNPreProdOctaveDepsList(); -***REMOVED*** case "Python": -***REMOVED*** return this.configuration.getSVNPreProdPythonDepsList(); -***REMOVED*** case "Pre-Installed": -***REMOVED*** return this.configuration.getSVNPreProdPreInstalledDepsList(); -***REMOVED*** case "Windows-compiled": -***REMOVED*** return this.configuration.getSVNPreProdWCDepsList(); -***REMOVED*** default: -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** - - - - - -***REMOVED*** public String getProdDependencyFile(String language) { -***REMOVED*** switch (language) { -***REMOVED*** case "R": -***REMOVED*** return this.configuration.getSVNProdCRANDepsList(); -***REMOVED*** case "R-blackbox": -***REMOVED*** return this.configuration.getSVNProdRBDepsList(); -***REMOVED*** case "Java": -***REMOVED*** return this.configuration.getSVNProdJavaDepsList(); -***REMOVED*** case "Knime-Workflow": -***REMOVED*** return this.configuration.getSVNProdKWDepsList(); -***REMOVED*** case "Linux-compiled": -***REMOVED*** return this.configuration.getSVNProdLinuxCompiledDepsList(); -***REMOVED*** case "Octave": -***REMOVED*** return this.configuration.getSVNProdOctaveDepsList(); -***REMOVED*** case "Python": -***REMOVED*** return this.configuration.getSVNProdPythonDepsList(); -***REMOVED*** case "Pre-Installed": -***REMOVED*** return this.configuration.getSVNProdPreInstalledDepsList(); -***REMOVED*** case "Windows-compiled": -***REMOVED*** return this.configuration.getSVNProdWCDepsList(); -***REMOVED*** default: -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** - - -***REMOVED*** public String getDevDependencyFile(String language) { -***REMOVED*** switch (language) { -***REMOVED*** case "R": -***REMOVED*** return this.configuration.getSVNDevCRANDepsList(); -***REMOVED*** case "R-blackbox": -***REMOVED*** return this.configuration.getSVNDevRBDepsList(); -***REMOVED*** case "Java": -***REMOVED*** return this.configuration.getSVNDevJavaDepsList(); -***REMOVED*** case "Knime-Workflow": -***REMOVED*** return this.configuration.getSVNDevKWDepsList(); -***REMOVED*** case "Linux-compiled": -***REMOVED*** return this.configuration.getSVNDevLinuxCompiledDepsList(); -***REMOVED*** case "Octave": -***REMOVED*** return this.configuration.getSVNDevOctaveDepsList(); -***REMOVED*** case "Python": -***REMOVED*** return this.configuration.getSVNDevPythonDepsList(); -***REMOVED*** case "Pre-Installed": -***REMOVED*** return this.configuration.getSVNDevPreInstalledDepsList(); -***REMOVED*** case "Windows-compiled": -***REMOVED*** return this.configuration.getSVNDevWCDepsList(); -***REMOVED*** default: -***REMOVED*** return null; -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** - public String getStagingDependencyFile(String language) { switch (language) { @@ -209,162 +123,32 @@ public class SVNUpdater { ***REMOVED*** - - - -***REMOVED*** public void readPreProdDeps(Algorithm algorithm) throws SVNException { -***REMOVED*** if (algorithm.getLanguage().equals("R")) { -***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Java")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) { -***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Octave")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Python")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** + public String getProdDependencyFile(String language) { + switch (language) { + case "R": + return this.configuration.getSVNProdCRANDepsList(); + case "R-blackbox": + return this.configuration.getSVNProdRBDepsList(); + case "Java": + return this.configuration.getSVNProdJavaDepsList(); + case "Knime-Workflow": + return this.configuration.getSVNProdKWDepsList(); + case "Linux-compiled": + return this.configuration.getSVNProdLinuxCompiledDepsList(); + case "Octave": + return this.configuration.getSVNProdOctaveDepsList(); + case "Python": + return this.configuration.getSVNProdPythonDepsList(); + case "Pre-Installed": + return this.configuration.getSVNProdPreInstalledDepsList(); + case "Windows-compiled": + return this.configuration.getSVNProdWCDepsList(); + default: + return null; + ***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** public void readRPRotoDeps(Algorithm algorithm) throws SVNException { -***REMOVED*** if (algorithm.getLanguage().equals("R")) { -***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Java")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) { -***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Octave")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Python")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** public void readProdDeps(Algorithm algorithm) throws SVNException { -***REMOVED*** if (algorithm.getLanguage().equals("R")) { -***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNProdCRANDepsList(), algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Java")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Octave")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Python")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** -***REMOVED*** public void readRDevDeps(Algorithm algorithm) throws SVNException { -***REMOVED*** if (algorithm.getLanguage().equals("R")) { -***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("R-blackbox")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Java")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Knime-Workflow")) { -***REMOVED*** this.checkIfAvaialable(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Linux-compiled")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Octave")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Python")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Windows-compiled")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED*** if (algorithm.getLanguage().equals("Pre-Installed")) { -***REMOVED*** this.checkIfAvaialable("", algorithm.getDependencies()); -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** - - - - -***REMOVED*** public void updateProdDeps(Algorithm algorithm) { -***REMOVED*** this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies()); -***REMOVED*** this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies()); -***REMOVED*** this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies()); -***REMOVED******REMOVED*** - -***REMOVED*** public void updateSVNAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) { -***REMOVED*** -***REMOVED*** if (env.equals("Dev")){ -***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNDevAlgorithmsList(), algorithm, targetVRE, category,algorithm_type, user, env); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Prod")){ -***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Proto")){ -***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env); -***REMOVED*** ***REMOVED*** -***REMOVED*** -***REMOVED*** if (env.equals("Preprod")){ -***REMOVED*** this.updateSVNAlgorithmList(this.configuration.getSVNPreProdAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user, env); -***REMOVED*** ***REMOVED*** -***REMOVED******REMOVED*** public void updateSVNStagingAlgorithmList(Algorithm algorithm, String targetVRE, String category, String algorithm_type, String user/*, String env*/) { this.updateSVNAlgorithmList(this.configuration.getSVNStagingAlgorithmsList(), algorithm, targetVRE, category, algorithm_type, user); @@ -381,6 +165,13 @@ public class SVNUpdater { ***REMOVED*** + public void updateAlgorithmFilesProd(File a) throws SVNException{ + ***REMOVED***this.updateAlgorithmList(this.configuration.getSVNMainAlgoRepo(), a); + this.updateAlgorithmList(this.configuration.getProdRepository(), a); +***REMOVED*** + + + public void updateAlgorithmList(String svnMainAlgoRepo, File a) throws SVNException { try { System.out.println("Adding .jar file: " + a + " to repository " + svnMainAlgoRepo); @@ -533,6 +324,7 @@ public class SVNUpdater { ***REMOVED*** ***REMOVED*** + public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String category, String algorithm_type/*,String env*/) throws ParseException { ***REMOVED***Timestamp timestamp = new Timestamp(System.currentTimeMillis()); ***REMOVED***long unixTime = System.currentTimeMillis() / 1000L; @@ -547,6 +339,9 @@ public class SVNUpdater { + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); sb.append("none | "); sb.append(this.getTimeZone() + " | "); + + System.out.println("Algo details: "+sb.toString()); + return sb.toString(); ***REMOVED*** @@ -746,7 +541,6 @@ public class SVNUpdater { SVNFileUtil.closeFile(checksumStream); ***REMOVED*** ***REMOVED*** - public List checkMatch(String[] lines, List ls) { Set ss = new HashSet(ls); ss.addAll(Arrays.asList(lines)); @@ -758,7 +552,7 @@ public class SVNUpdater { Calendar cal = Calendar.getInstance(); cal.getTime(); DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy"); - Date fromDate = (Date)formatter.parse(cal.getTime().toString()); + Date fromDate = formatter.parse(cal.getTime().toString()); TimeZone central = TimeZone.getTimeZone("UTC"); formatter.setTimeZone(central); System.out.println(formatter.format(fromDate)); @@ -782,7 +576,7 @@ public class SVNUpdater { Calendar cal = Calendar.getInstance(); cal.getTime(); DateFormat formatter = new SimpleDateFormat("EEE MMM dd HH:mm:ss zzz yyyy"); - Date fromDate = (Date)formatter.parse(cal.getTime().toString()); + Date fromDate = formatter.parse(cal.getTime().toString()); TimeZone central = TimeZone.getTimeZone("UTC"); formatter.setTimeZone(central); System.out.println(formatter.format(fromDate)); diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java index ed15ed6..929ed12 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/AlgorithmPackageParserTest.java @@ -114,6 +114,9 @@ public class AlgorithmPackageParserTest { ***REMOVED*** "/home/ngalante/.proxy-settings")); new AlgorithmPackageParserTest().extractAllAlgorithms(); +***REMOVED*** AlgorithmPackageParserTest at = new AlgorithmPackageParserTest(); +***REMOVED*** Algorithm a = at.extractAlgorithm("http:***REMOVED***data.d4science.org/YjJ3TmJab1dqYzVoTmppdjlsK0l0b1ZXWGtzWlQ1NHNHbWJQNStIS0N6Yz0"); +***REMOVED*** System.out.println(a.getFullname()); ***REMOVED*** ***REMOVED***