From 0bc11c4ceeeafc474c474bceb100b148b62bf75b Mon Sep 17 00:00:00 2001 From: Gabriele Giammatteo Date: Wed, 17 May 2017 16:16:25 +0000 Subject: [PATCH] updated SVNUpdater git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148811 82a268e6-3cf1-43bd-a215-b396298e98cf --- pom.xml | 272 ++++++++++-------- .../poolmanager/datamodel/Algorithm.java | 32 +++ .../poolmanager/service/DMPMJob.java | 44 +-- .../service/DataminerPoolManager.java | 9 +- .../service/ProductionPublishingJob.java | 12 +- .../poolmanager/service/StagingJob.java | 21 +- .../poolmanager/util/ClusterBuilder.java | 2 +- .../dataminer/poolmanager/util/Props.java | 103 ------- .../poolmanager/util/SVNUpdater.java | 115 +++++++- .../util/ServiceConfiguration.java | 91 ++++++ src/main/resources/service.properties | 14 + src/main/webapp/WEB-INF/gcube-app.xml | 4 +- src/main/webapp/WEB-INF/web.xml | 24 +- .../dataminerpoolmanager/JobTest.java | 5 +- .../dataminerpoolmanager/SVNTests.java | 26 ++ .../dataminerpoolmanager/SVNUpdaterTest.java | 25 -- 16 files changed, 491 insertions(+), 308 deletions(-) delete mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/Props.java create mode 100644 src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java create mode 100644 src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java delete mode 100644 src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java diff --git a/pom.xml b/pom.xml index 5232d15..0198b42 100644 --- a/pom.xml +++ b/pom.xml @@ -1,141 +1,169 @@ - 4.0.0 + xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd"> + 4.0.0 - - maven-parent - org.gcube.tools - 1.0.0 - - + + maven-parent + org.gcube.tools + 1.0.0 + + - org.gcube.dataAnalysis - dataminer-pool-manager - 2.0.0-SNAPSHOT - war + org.gcube.dataAnalysis + dataminer-pool-manager + 2.0.0-SNAPSHOT + war - dataminer-pool-manager - - + dataminer-pool-manager + + - - distro - UTF-8 - + + distro + UTF-8 + 1.5.13 + 2.25.1 + - - - org.gcube.core - common-smartgears - - provided - - - commons-io - commons-io - [2.5.0,2.6.0) - - - org.antlr - stringtemplate - [4.0.0, 4.1.0) - - - org.tmatesoft.svnkit - svnkit - 1.8.5 - - - commons-configuration - commons-configuration - 1.10 - + - - net.sf.opencsv - opencsv - 2.3 - - - org.gcube.resources.discovery - ic-client - provided - - - org.yaml - snakeyaml - 1.16 - - - org.glassfish.jersey.media - jersey-media-json-jackson - 2.23.1 - - - org.glassfish.jersey.core - jersey-client - 2.22.1 - - - org.glassfish.jersey.containers - jersey-container-servlet - 2.22.1 - + + org.gcube.core + common-smartgears + provided + - - org.slf4j - slf4j-api - provided - + + org.gcube.resources.discovery + ic-client + provided + - - - - - org.gcube.distribution - maven-smartgears-bom - LATEST - pom - import - - - + + commons-io + commons-io + [2.5.0,2.6.0) + + + org.antlr + stringtemplate + [4.0.0, 4.1.0) + + + org.tmatesoft.svnkit + svnkit + 1.8.5 + + + commons-configuration + commons-configuration + 1.10 + - - + + net.sf.opencsv + opencsv + 2.3 + - - maven-compiler-plugin - - 1.7 - 1.7 - - + + org.yaml + snakeyaml + 1.16 + + + org.glassfish.jersey.media + jersey-media-json-jackson + $***REMOVED***version.jersey***REMOVED*** + - - org.apache.maven.plugins - maven-assembly-plugin - - - $***REMOVED***distroDirectory***REMOVED***/descriptor.xml - - - - - servicearchive - install - - single - - - - + + org.glassfish.jersey.core + jersey-client + $***REMOVED***version.jersey***REMOVED*** + - - + + org.glassfish.jersey.containers + jersey-container-servlet + $***REMOVED***version.jersey***REMOVED*** + + + + + org.slf4j + slf4j-api + provided + + + + + + + + + org.gcube.distribution + maven-smartgears-bom + LATEST + pom + import + + + + + + + + + org.apache.maven.plugins + maven-assembly-plugin + + + $***REMOVED***distroDirectory***REMOVED***/descriptor.xml + + + + + servicearchive + install + + single + + + + + + + diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java index 60b9cb8..b4b616a 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java @@ -1,6 +1,8 @@ package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; import java.util.Collection; +import java.util.HashSet; +***REMOVED*** import java.util.Vector; @@ -76,6 +78,36 @@ public class Algorithm ***REMOVED*** public Collection getDependencies() ***REMOVED*** return dependencies; ***REMOVED*** + + public Collection getOSDependencies()***REMOVED*** + Set deps = new HashSet<>(); + for(Dependency d: this.getDependencies())***REMOVED*** + if(d.getType().equals("os"))***REMOVED*** + deps.add(d); + ***REMOVED*** + ***REMOVED*** + return deps; + ***REMOVED*** + + public Collection getCranDependencies()***REMOVED*** + Set deps = new HashSet<>(); + for(Dependency d: this.getDependencies())***REMOVED*** + if(d.getType().equals("cran"))***REMOVED*** + deps.add(d); + ***REMOVED*** + ***REMOVED*** + return deps; + ***REMOVED*** + + public Collection getGitHubDependencies()***REMOVED*** + Set deps = new HashSet<>(); + for(Dependency d: this.getDependencies())***REMOVED*** + if(d.getType().equals("github"))***REMOVED*** + deps.add(d); + ***REMOVED*** + ***REMOVED*** + return deps; + ***REMOVED*** public void setDependencies(Collection deps) ***REMOVED*** this.dependencies = deps; diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java index aa69411..4d7f9a4 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java @@ -74,27 +74,27 @@ public abstract class DMPMJob ***REMOVED*** PrintStream ps = new PrintStream(fos); return worker.execute(ps); ***REMOVED*** - - - protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED*** - for (Dependency d : algo.getDependencies()) ***REMOVED*** - - if (d.getType().equals("os")) ***REMOVED*** - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("cran")) ***REMOVED*** - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls); - ***REMOVED*** - if (d.getType().equals("github")) ***REMOVED*** - List ls = new LinkedList(); - ls.add(d.getName()); - this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls); - ***REMOVED*** - ***REMOVED*** -***REMOVED*** + +***REMOVED*** +***REMOVED*** protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED*** +***REMOVED*** for (Dependency d : algo.getDependencies()) ***REMOVED*** +***REMOVED*** +***REMOVED*** if (d.getType().equals("os")) ***REMOVED*** +***REMOVED*** List ls = new LinkedList(); +***REMOVED*** ls.add(d.getName()); +***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls); +***REMOVED*** ***REMOVED*** +***REMOVED*** if (d.getType().equals("cran")) ***REMOVED*** +***REMOVED*** List ls = new LinkedList(); +***REMOVED*** ls.add(d.getName()); +***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls); +***REMOVED*** ***REMOVED*** +***REMOVED*** if (d.getType().equals("github")) ***REMOVED*** +***REMOVED*** List ls = new LinkedList(); +***REMOVED*** ls.add(d.getName()); +***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls); +***REMOVED*** ***REMOVED*** +***REMOVED*** ***REMOVED*** +***REMOVED******REMOVED*** ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index 98f2d6c..f3ad2ec 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -5,10 +5,11 @@ import java.io.FileNotFoundException; ***REMOVED*** import java.util.Scanner; +***REMOVED*** ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; -import org.gcube.dataanalysis.dataminer.poolmanager.util.Props; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; ***REMOVED*** @@ -20,7 +21,7 @@ public class DataminerPoolManager ***REMOVED*** public DataminerPoolManager() ***REMOVED*** try ***REMOVED*** ***REMOVED***TODO: read this from configuration - this.svnUpdater = new SVNUpdater(new Props().getSVNrepo()); + this.svnUpdater = new SVNUpdater(new ServiceConfiguration()); ***REMOVED*** catch (SVNException e) ***REMOVED*** e.printStackTrace(); ***REMOVED*** @@ -33,7 +34,7 @@ public class DataminerPoolManager ***REMOVED*** Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); - DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster); + DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get()); String id = job.start(); return id; ***REMOVED*** @@ -42,7 +43,7 @@ public class DataminerPoolManager ***REMOVED*** Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); - DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster); + DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE); String id = job.start(); return id; ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java index ca3c1d5..359aefa 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java @@ -8,20 +8,24 @@ public class ProductionPublishingJob extends DMPMJob ***REMOVED*** private Algorithm algorithm; private Cluster prodCluster; + private String targetVREName; - public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster) ***REMOVED*** + public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName) ***REMOVED*** super(svnUpdater); this.algorithm = algorithm; this.prodCluster = prodCluster; + this.targetVREName = targetVREName; ***REMOVED*** ***REMOVED*** protected void execute() ***REMOVED*** try ***REMOVED*** - this.updateSVNDependencies(this.algorithm, false); + this.svnUpdater.updateProdDeps(this.algorithm); - int ret = this.executeAnsibleWorker( - createWorker(this.algorithm, this.prodCluster, false, "gcube")); + this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod"); + +***REMOVED*** int ret = this.executeAnsibleWorker( +***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube")); ***REMOVED*** catch (Exception e) ***REMOVED*** e.printStackTrace(); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java index b958a0e..1567dad 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java @@ -10,12 +10,19 @@ public class StagingJob extends DMPMJob ***REMOVED*** private Algorithm algorithm; private Cluster stagingCluster; private Cluster rProtoCluster; + private String rProtoVREName; - public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster) ***REMOVED*** + public StagingJob( + SVNUpdater svnUpdater, + Algorithm algorithm, + Cluster stagingCluster, + Cluster rProtoCluster, + String rProtoVREName) ***REMOVED*** super(svnUpdater); this.algorithm = algorithm; this.stagingCluster = stagingCluster; this.rProtoCluster = rProtoCluster; + this.rProtoVREName = rProtoVREName; ***REMOVED*** ***REMOVED*** @@ -26,13 +33,13 @@ public class StagingJob extends DMPMJob ***REMOVED*** createWorker(this.algorithm, this.stagingCluster, true, "root")); if(ret == 0)***REMOVED*** + + this.svnUpdater.updateRPRotoDeps(this.algorithm); + + this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, "DataMiner Pool Manager", "Proto"); - - this.updateSVNDependencies(this.algorithm, true); - - - int ret2 = this.executeAnsibleWorker( - createWorker(this.algorithm, this.rProtoCluster, false, "gcube")); +***REMOVED*** int ret2 = this.executeAnsibleWorker( +***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, false, "gcube")); ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java index c96df6b..46d21f9 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java @@ -15,7 +15,7 @@ public class ClusterBuilder ***REMOVED*** ***REMOVED***1. to complete public static Cluster getStagingDataminerCluster() throws FileNotFoundException***REMOVED*** Cluster cluster = new Cluster(); - Props p = new Props(); + ServiceConfiguration p = new ServiceConfiguration(); Host h = new Host(p.getStagingHost()); ***REMOVED***TODO: read this from configuration or IS? h.setName(p.getStagingHost()); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/Props.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/Props.java deleted file mode 100644 index 033f7e7..0000000 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/Props.java +++ /dev/null @@ -1,103 +0,0 @@ -package org.gcube.dataanalysis.dataminer.poolmanager.util; - -import java.io.FileInputStream; -import java.io.FileNotFoundException; -***REMOVED*** -import java.io.InputStream; -import java.util.Properties; - -public class Props ***REMOVED*** - public static String home = System.getProperty("user.home"); - - public Props() ***REMOVED*** -***REMOVED*** - - public String getCSVUrl() ***REMOVED*** - Properties props = new Properties(); - ***REMOVED***read from war - ***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties"); - - ***REMOVED***read from filesystem - FileInputStream input; - try ***REMOVED*** - input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties"); - ***REMOVED*** loading properites from properties file - try ***REMOVED*** - props.load(input); - ***REMOVED*** catch (IOException e) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e1.printStackTrace(); - ***REMOVED*** - ***REMOVED*** reading propeprty - String path = props.getProperty("HAPROXY_CSV"); - return path; - -***REMOVED*** - - - public String getSVNrepo() ***REMOVED*** - Properties props = new Properties(); - ***REMOVED***read from war - ***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties"); - - ***REMOVED***read from filesystem - FileInputStream input; - try ***REMOVED*** - input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties"); - ***REMOVED*** loading properites from properties file - try ***REMOVED*** - props.load(input); - ***REMOVED*** catch (IOException e) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e1.printStackTrace(); - ***REMOVED*** - ***REMOVED*** reading propeprty - String path = props.getProperty("SVN_REPO"); - return path; - -***REMOVED*** - - - - public String getStagingHost() ***REMOVED*** - Properties props = new Properties(); - ***REMOVED***read from war - ***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties"); - - ***REMOVED***read from filesystem - FileInputStream input; - try ***REMOVED*** - input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties"); - ***REMOVED*** loading properites from properties file - try ***REMOVED*** - props.load(input); - ***REMOVED*** catch (IOException e) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e.printStackTrace(); - ***REMOVED*** - ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED*** - ***REMOVED*** TODO Auto-generated catch block - e1.printStackTrace(); - ***REMOVED*** - - ***REMOVED*** reading propeprty - String path = props.getProperty("STAGING_HOST"); - return path; - -***REMOVED*** - - public static void main(String[] args) throws FileNotFoundException ***REMOVED*** - Props a = new Props(); - System.out.println(new Props().getSVNrepo()); - System.out.println(a.getStagingHost()); - System.out.println(a.getCSVUrl()); -***REMOVED*** -***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java index c40d615..232588c 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java @@ -1,5 +1,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util; +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency; ***REMOVED*** import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; @@ -21,32 +23,117 @@ import java.util.*; */ public class SVNUpdater ***REMOVED*** - private SVNRepository svnRepository; + private ServiceConfiguration configuration; - public SVNUpdater(String rootURL) throws SVNException ***REMOVED*** - this.svnRepository = this.getSvnRepository( - rootURL); - ***REMOVED*** - - - private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED*** - SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); + public SVNUpdater(ServiceConfiguration configuration) throws SVNException ***REMOVED*** + this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository())); ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); - repository.setAuthenticationManager(authManager); - System.out.println(repository.getLocation()); - - return repository; + this.svnRepository.setAuthenticationManager(authManager); + this.configuration = configuration; ***REMOVED*** + public void updateRPRotoDeps(Algorithm algorithm)***REMOVED*** + this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies()); + this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies()); + this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies()); + ***REMOVED*** - public void updateSVN(String file, List ldep) ***REMOVED*** + public void updateProdDeps(Algorithm algorithm)***REMOVED*** + this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies()); + this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies()); + this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies()); + ***REMOVED*** + + + public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED*** + this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env); + ***REMOVED*** + + + public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED*** + this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env); + ***REMOVED*** + + public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env)***REMOVED*** try ***REMOVED*** + System.out.println("Updating algorithm list: "+ file); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + + ***REMOVED***check if the algorithm is already in the list (match the class name) + for(String l: lines)***REMOVED*** + if(l.contains(algorithm.getClazz()))***REMOVED*** + return; + ***REMOVED*** + ***REMOVED*** + + ***REMOVED***the algorithm is not in the list. Add it + List newContent = new LinkedList<>(Arrays.asList(lines)); + newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, user, env)); + Collections.sort(newContent); + + + final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator(); + + byte[] originalContents = byteArrayOutputStream.toByteArray(); + + final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null); + commitEditor.openRoot(-1); + commitEditor.openFile(file, -1); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + for (String line : newContent) ***REMOVED*** + baos.write(line.getBytes()); + baos.write("\n".getBytes()); + ***REMOVED*** + byte[] bytes = baos.toByteArray(); + + commitEditor.applyTextDelta(file,md5(originalContents)); + + final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0, + new ByteArrayInputStream(bytes), commitEditor, true); + commitEditor.closeFile(file, checksum); + commitEditor.closeEdit(); + + + ***REMOVED***catch(Exception ex)***REMOVED*** + ex.printStackTrace(); + ***REMOVED*** + + finally ***REMOVED*** + svnRepository.closeSession(); + ***REMOVED*** ***REMOVED*** + + + public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED*** + StringBuffer sb = new StringBuffer("| "); + sb.append(algorithm.getName() +" | "); + sb.append(user + " | "); + sb.append(algorithm.getCategory() + " | "); + sb.append(env + " | "); + sb.append("./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" | "); + sb.append("none |"); + return sb.toString(); + ***REMOVED*** + + + public void updateSVN(String file, Collection deps) ***REMOVED*** + try ***REMOVED*** + System.out.println("Updating dependencies list: "+ file); + + final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); + String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); + + List ldep = new LinkedList<>(); + for(Dependency d: deps)***REMOVED*** + ldep.add(d.getName()); + ***REMOVED*** List aa = this.checkMatch(lines, ldep); Collections.sort(aa); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java new file mode 100644 index 0000000..954baac --- /dev/null +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java @@ -0,0 +1,91 @@ +package org.gcube.dataanalysis.dataminer.poolmanager.util; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +***REMOVED*** +import java.util.Properties; + +public class ServiceConfiguration ***REMOVED*** + public static String home = System.getProperty("user.home"); + + private Properties props; + + public ServiceConfiguration() ***REMOVED*** + this(home+"/dataminer-pool-manager/dpmConfig/service.properties"); +***REMOVED*** + + public ServiceConfiguration(String configFile) ***REMOVED*** + this.props = new Properties(); + + FileInputStream input; + + try ***REMOVED*** + input = new FileInputStream(configFile); + ***REMOVED*** loading properites from properties file + try ***REMOVED*** + props.load(input); + ***REMOVED*** catch (IOException e) ***REMOVED*** + ***REMOVED*** TODO Auto-generated catch block + e.printStackTrace(); + ***REMOVED*** + ***REMOVED*** catch (FileNotFoundException e1) ***REMOVED*** + ***REMOVED*** TODO Auto-generated catch block + e1.printStackTrace(); + ***REMOVED*** +***REMOVED*** + + + + public String getSVNRepository()***REMOVED*** + return props.getProperty("svn.repository"); +***REMOVED*** + + public String getSVNRProtoAlgorithmsList()***REMOVED*** + return props.getProperty("svn.rproto.algorithms-list"); +***REMOVED*** + + public String getSVNRProtoOSDepsList()***REMOVED*** + return props.getProperty("svn.rproto.deps-os"); +***REMOVED*** + + public String getSVNRProtoCRANDepsList()***REMOVED*** + return props.getProperty("svn.rproto.deps-cran"); +***REMOVED*** + + public String getSVNRProtoGitHubDepsList()***REMOVED*** + return props.getProperty("svn.rproto.deps-github"); +***REMOVED*** + + public String getSVNProdAlgorithmsList()***REMOVED*** + return props.getProperty("svn.prod.algorithms-list"); +***REMOVED*** + + public String getSVNProdOSDepsList()***REMOVED*** + return props.getProperty("svn.prod.deps-os"); +***REMOVED*** + + public String getSVNRProdCRANDepsList()***REMOVED*** + return props.getProperty("svn.prod.deps-cran"); +***REMOVED*** + + public String getSVNRProdGitHubDepsList()***REMOVED*** + return props.getProperty("svn.prod.deps-github"); +***REMOVED*** + + public String getCSVUrl() ***REMOVED*** + return props.getProperty("HAPROXY_CSV"); +***REMOVED*** + + + public String getStagingHost() ***REMOVED*** + return props.getProperty("STAGING_HOST"); +***REMOVED*** + + public static void main(String[] args) throws FileNotFoundException ***REMOVED*** + ServiceConfiguration a = new ServiceConfiguration(); + System.out.println(a.getStagingHost()); + System.out.println(a.getCSVUrl()); +***REMOVED*** + + +***REMOVED*** diff --git a/src/main/resources/service.properties b/src/main/resources/service.properties index 915900b..498c796 100644 --- a/src/main/resources/service.properties +++ b/src/main/resources/service.properties @@ -3,3 +3,17 @@ STAGING_HOST: dataminer1-devnext.d4science.org SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ #HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0 + +svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube + +svn.rproto.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/proto/algorithms +svn.rproto.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_deb_pkgs.txt +svn.rproto.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_cran_pkgs.txt +svn.rproto.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_github_pkgs.txt + + +svn.prod.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/prod/algorithms +svn.prod.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt +svn.prod.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt +svn.prod.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt + diff --git a/src/main/webapp/WEB-INF/gcube-app.xml b/src/main/webapp/WEB-INF/gcube-app.xml index 15f231c..204fb09 100644 --- a/src/main/webapp/WEB-INF/gcube-app.xml +++ b/src/main/webapp/WEB-INF/gcube-app.xml @@ -9,5 +9,7 @@ - + + /api/swagger.* + \ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index 2a231cb..5596e2b 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -11,14 +11,32 @@ org.glassfish.jersey.servlet.ServletContainer jersey.config.server.provider.packages - org.gcube.dataanalysis.dataminer.poolmanager.rest + + io.swagger.jaxrs.listing, + org.gcube.dataanalysis.dataminer.poolmanager.rest 1 REST-API - /rest/* + /api/* - + + + diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java index 7d2e20b..096d8d1 100644 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java @@ -11,6 +11,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.service.StagingJob; ***REMOVED*** import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration; ***REMOVED*** public class JobTest ***REMOVED*** @@ -19,13 +20,13 @@ public class JobTest ***REMOVED*** ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); - SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); + SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration()); Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); ***REMOVED***test phase Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); - DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster); + DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get()); job.start(); ***REMOVED***release phase diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java new file mode 100644 index 0000000..54a82f5 --- /dev/null +++ b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java @@ -0,0 +1,26 @@ +package org.gcube.dataanalysis.dataminerpoolmanager; + +***REMOVED*** +***REMOVED*** +import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; +import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration; +***REMOVED*** + +***REMOVED*** + +/** + * Created by ggiammat on 5/17/17. + */ +public class SVNTests ***REMOVED*** + + + public static void main(String[] args) throws SVNException, IOException, InterruptedException ***REMOVED*** + + + SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ggiammat/tmp/dmpm.properties")); + Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); + + + svnUpdater.updateRPRotoDeps(algo); + ***REMOVED*** +***REMOVED*** diff --git a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java b/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java deleted file mode 100644 index d9c4d04..0000000 --- a/src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java +++ /dev/null @@ -1,25 +0,0 @@ -package org.gcube.dataanalysis.dataminerpoolmanager; - -***REMOVED*** -import java.util.LinkedList; -***REMOVED*** - -***REMOVED*** -import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; -***REMOVED*** - -public class SVNUpdaterTest ***REMOVED*** - - public static void main(String[] args) throws SVNException, IOException ***REMOVED*** - ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); - - SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); - - String test = "testDep"; - List ldep = new LinkedList<>(); - ldep.add(test); - - svnUpdater.updateSVN("test_r_cran_pkgs.txt", ldep); -***REMOVED*** - -***REMOVED***