gCubeSystem
/
dataminer-pool-manager
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

updated SVNUpdater 

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148811 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gabriele Giammatteo 2017-05-17 16:16:25 +00:00
parent 8a5693f6b2
commit 04c9b93715
16 changed files with 491 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

272
pom.xml
View File

@ -1,141 +1,169 @@
<project xmlns="http:***REMOVED***maven.apache.org/POM/4.0.0" xmlns:xsi="http:***REMOVED***www.w3.org/2001/XMLSchema-instance" <project xmlns="http:***REMOVED***maven.apache.org/POM/4.0.0" xmlns:xsi="http:***REMOVED***www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd"> xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<parent> <parent>
<artifactId>maven-parent</artifactId> <artifactId>maven-parent</artifactId>
<groupId>org.gcube.tools</groupId> <groupId>org.gcube.tools</groupId>
<version>1.0.0</version> <version>1.0.0</version>
<relativePath /> <relativePath/>
</parent> </parent>
<groupId>org.gcube.dataAnalysis</groupId> <groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId> <artifactId>dataminer-pool-manager</artifactId>
<version>2.0.0-SNAPSHOT</version> <version>2.0.0-SNAPSHOT</version>
<packaging>war</packaging> <packaging>war</packaging>
<name>dataminer-pool-manager</name> <name>dataminer-pool-manager</name>
<description> <description>
</description> </description>
<properties> <properties>
<distroDirectory>distro</distroDirectory> <distroDirectory>distro</distroDirectory>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties> <version.swagger>1.5.13</version.swagger>
<version.jersey>2.25.1</version.jersey>
</properties>
<dependencies> <dependencies>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version> -->
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>[2.5.0,2.6.0)</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>stringtemplate</artifactId>
<version>[4.0.0, 4.1.0)</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.5</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
<artifactId>ic-client</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>2.23.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>2.22.1</version>
</dependency>
<dependency> <dependency>
<groupId>org.glassfish.jersey.containers</groupId> <groupId>org.gcube.core</groupId>
<artifactId>jersey-container-servlet</artifactId> <artifactId>common-smartgears</artifactId>
<version>2.22.1</version> <scope>provided</scope>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.slf4j</groupId> <groupId>org.gcube.resources.discovery</groupId>
<artifactId>slf4j-api</artifactId> <artifactId>ic-client</artifactId>
<scope>provided</scope> <scope>provided</scope>
</dependency> </dependency>
</dependencies>
<dependencyManagement> <dependency>
<dependencies> <groupId>commons-io</groupId>
<dependency> <artifactId>commons-io</artifactId>
<groupId>org.gcube.distribution</groupId> <version>[2.5.0,2.6.0)</version>
<artifactId>maven-smartgears-bom</artifactId> </dependency>
<version>LATEST</version> <dependency>
<type>pom</type> <groupId>org.antlr</groupId>
<scope>import</scope> <artifactId>stringtemplate</artifactId>
</dependency> <version>[4.0.0, 4.1.0)</version>
</dependencies> </dependency>
</dependencyManagement> <dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.5</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<build> <dependency>
<plugins> <groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<plugin> <dependency>
<artifactId>maven-compiler-plugin</artifactId> <groupId>org.yaml</groupId>
<configuration> <artifactId>snakeyaml</artifactId>
<source>1.7</source> <version>1.16</version>
<target>1.7</target> </dependency>
</configuration> <dependency>
</plugin> <groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<plugin> <dependency>
<groupId>org.apache.maven.plugins</groupId> <groupId>org.glassfish.jersey.core</groupId>
<artifactId>maven-assembly-plugin</artifactId> <artifactId>jersey-client</artifactId>
<configuration> <version>${version.jersey***REMOVED***</version>
<descriptors> </dependency>
<descriptor>${distroDirectory***REMOVED***/descriptor.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>servicearchive</id>
<phase>install</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins> <dependency>
</build> <groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<!--
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-jersey2-jaxrs</artifactId>
<version>${version.swagger***REMOVED***</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
<version>${version.swagger***REMOVED***</version>
</dependency>
-->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-smartgears-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>${distroDirectory***REMOVED***/descriptor.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>servicearchive</id>
<phase>install</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project> </project>

32
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/datamodel/Algorithm.java
View File

@ -1,6 +1,8 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel; package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection; import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.Vector; import java.util.Vector;
@ -77,6 +79,36 @@ public class Algorithm {
return dependencies; return dependencies;
***REMOVED*** ***REMOVED***
public Collection<Dependency> getOSDependencies(){
Set<Dependency> deps = new HashSet<>();
for(Dependency d: this.getDependencies()){
if(d.getType().equals("os")){
deps.add(d);
***REMOVED***
***REMOVED***
return deps;
***REMOVED***
public Collection<Dependency> getCranDependencies(){
Set<Dependency> deps = new HashSet<>();
for(Dependency d: this.getDependencies()){
if(d.getType().equals("cran")){
deps.add(d);
***REMOVED***
***REMOVED***
return deps;
***REMOVED***
public Collection<Dependency> getGitHubDependencies(){
Set<Dependency> deps = new HashSet<>();
for(Dependency d: this.getDependencies()){
if(d.getType().equals("github")){
deps.add(d);
***REMOVED***
***REMOVED***
return deps;
***REMOVED***
public void setDependencies(Collection<Dependency> deps) { public void setDependencies(Collection<Dependency> deps) {
this.dependencies = deps; this.dependencies = deps;
***REMOVED*** ***REMOVED***

40
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DMPMJob.java
View File

@ -75,26 +75,26 @@ public abstract class DMPMJob {
return worker.execute(ps); return worker.execute(ps);
***REMOVED*** ***REMOVED***
protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
for (Dependency d : algo.getDependencies()) {
if (d.getType().equals("os")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("github")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
***REMOVED***
***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED*** protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
***REMOVED*** for (Dependency d : algo.getDependencies()) {
***REMOVED***
***REMOVED*** if (d.getType().equals("os")) {
***REMOVED*** List<String> ls = new LinkedList<String>();
***REMOVED*** ls.add(d.getName());
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
***REMOVED*** if (d.getType().equals("cran")) {
***REMOVED*** List<String> ls = new LinkedList<String>();
***REMOVED*** ls.add(d.getName());
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
***REMOVED*** if (d.getType().equals("github")) {
***REMOVED*** List<String> ls = new LinkedList<String>();
***REMOVED*** ls.add(d.getName());
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** ***REMOVED***

9
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -5,10 +5,11 @@ import java.io.FileNotFoundException;
***REMOVED*** ***REMOVED***
import java.util.Scanner; import java.util.Scanner;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm; import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED*** ***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.Props; import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
@ -20,7 +21,7 @@ public class DataminerPoolManager {
public DataminerPoolManager() { public DataminerPoolManager() {
try { try {
***REMOVED***TODO: read this from configuration ***REMOVED***TODO: read this from configuration
this.svnUpdater = new SVNUpdater(new Props().getSVNrepo()); this.svnUpdater = new SVNUpdater(new ServiceConfiguration());
***REMOVED*** catch (SVNException e) { ***REMOVED*** catch (SVNException e) {
e.printStackTrace(); e.printStackTrace();
***REMOVED*** ***REMOVED***
@ -33,7 +34,7 @@ public class DataminerPoolManager {
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster); DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get());
String id = job.start(); String id = job.start();
return id; return id;
***REMOVED*** ***REMOVED***
@ -42,7 +43,7 @@ public class DataminerPoolManager {
Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE); Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster); DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE);
String id = job.start(); String id = job.start();
return id; return id;
***REMOVED*** ***REMOVED***

12
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/ProductionPublishingJob.java
View File

@ -8,20 +8,24 @@ public class ProductionPublishingJob extends DMPMJob {
private Algorithm algorithm; private Algorithm algorithm;
private Cluster prodCluster; private Cluster prodCluster;
private String targetVREName;
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster) { public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName) {
super(svnUpdater); super(svnUpdater);
this.algorithm = algorithm; this.algorithm = algorithm;
this.prodCluster = prodCluster; this.prodCluster = prodCluster;
this.targetVREName = targetVREName;
***REMOVED*** ***REMOVED***
@Override @Override
protected void execute() { protected void execute() {
try { try {
this.updateSVNDependencies(this.algorithm, false); this.svnUpdater.updateProdDeps(this.algorithm);
int ret = this.executeAnsibleWorker( this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
createWorker(this.algorithm, this.prodCluster, false, "gcube"));
***REMOVED*** int ret = this.executeAnsibleWorker(
***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube"));
***REMOVED*** catch (Exception e) { ***REMOVED*** catch (Exception e) {
e.printStackTrace(); e.printStackTrace();

17
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/StagingJob.java
View File

@ -10,12 +10,19 @@ public class StagingJob extends DMPMJob {
private Algorithm algorithm; private Algorithm algorithm;
private Cluster stagingCluster; private Cluster stagingCluster;
private Cluster rProtoCluster; private Cluster rProtoCluster;
private String rProtoVREName;
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster) { public StagingJob(
SVNUpdater svnUpdater,
Algorithm algorithm,
Cluster stagingCluster,
Cluster rProtoCluster,
String rProtoVREName) {
super(svnUpdater); super(svnUpdater);
this.algorithm = algorithm; this.algorithm = algorithm;
this.stagingCluster = stagingCluster; this.stagingCluster = stagingCluster;
this.rProtoCluster = rProtoCluster; this.rProtoCluster = rProtoCluster;
this.rProtoVREName = rProtoVREName;
***REMOVED*** ***REMOVED***
@Override @Override
@ -27,12 +34,12 @@ public class StagingJob extends DMPMJob {
if(ret == 0){ if(ret == 0){
this.svnUpdater.updateRPRotoDeps(this.algorithm);
this.updateSVNDependencies(this.algorithm, true); this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, "DataMiner Pool Manager", "Proto");
***REMOVED*** int ret2 = this.executeAnsibleWorker(
int ret2 = this.executeAnsibleWorker( ***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
***REMOVED*** ***REMOVED***

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ClusterBuilder.java
View File

@ -15,7 +15,7 @@ public class ClusterBuilder {
***REMOVED***1. to complete ***REMOVED***1. to complete
public static Cluster getStagingDataminerCluster() throws FileNotFoundException{ public static Cluster getStagingDataminerCluster() throws FileNotFoundException{
Cluster cluster = new Cluster(); Cluster cluster = new Cluster();
Props p = new Props(); ServiceConfiguration p = new ServiceConfiguration();
Host h = new Host(p.getStagingHost()); Host h = new Host(p.getStagingHost());
***REMOVED***TODO: read this from configuration or IS? ***REMOVED***TODO: read this from configuration or IS?
h.setName(p.getStagingHost()); h.setName(p.getStagingHost());

103
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/Props.java
View File

@ -1,103 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
***REMOVED***
import java.io.InputStream;
import java.util.Properties;
public class Props {
public static String home = System.getProperty("user.home");
public Props() {
***REMOVED***
public String getCSVUrl() {
Properties props = new Properties();
***REMOVED***read from war
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
***REMOVED***read from filesystem
FileInputStream input;
try {
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
***REMOVED*** loading properites from properties file
try {
props.load(input);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (FileNotFoundException e1) {
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
***REMOVED*** reading propeprty
String path = props.getProperty("HAPROXY_CSV");
return path;
***REMOVED***
public String getSVNrepo() {
Properties props = new Properties();
***REMOVED***read from war
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
***REMOVED***read from filesystem
FileInputStream input;
try {
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
***REMOVED*** loading properites from properties file
try {
props.load(input);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (FileNotFoundException e1) {
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
***REMOVED*** reading propeprty
String path = props.getProperty("SVN_REPO");
return path;
***REMOVED***
public String getStagingHost() {
Properties props = new Properties();
***REMOVED***read from war
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
***REMOVED***read from filesystem
FileInputStream input;
try {
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
***REMOVED*** loading properites from properties file
try {
props.load(input);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (FileNotFoundException e1) {
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
***REMOVED*** reading propeprty
String path = props.getProperty("STAGING_HOST");
return path;
***REMOVED***
public static void main(String[] args) throws FileNotFoundException {
Props a = new Props();
System.out.println(new Props().getSVNrepo());
System.out.println(a.getStagingHost());
System.out.println(a.getCSVUrl());
***REMOVED***
***REMOVED***

115
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNUpdater.java
View File

@ -1,5 +1,7 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util; package org.gcube.dataanalysis.dataminer.poolmanager.util;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL; import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager; import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
@ -21,32 +23,117 @@ import java.util.*;
*/ */
public class SVNUpdater { public class SVNUpdater {
private SVNRepository svnRepository; private SVNRepository svnRepository;
private ServiceConfiguration configuration;
public SVNUpdater(String rootURL) throws SVNException { public SVNUpdater(ServiceConfiguration configuration) throws SVNException {
this.svnRepository = this.getSvnRepository( this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
rootURL);
***REMOVED***
private SVNRepository getSvnRepository(String url) throws SVNException {
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
repository.setAuthenticationManager(authManager); this.svnRepository.setAuthenticationManager(authManager);
System.out.println(repository.getLocation()); this.configuration = configuration;
return repository;
***REMOVED*** ***REMOVED***
public void updateRPRotoDeps(Algorithm algorithm){
this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies());
this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies());
this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
***REMOVED***
public void updateSVN(String file, List<String> ldep) { public void updateProdDeps(Algorithm algorithm){
this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
***REMOVED***
public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env){
this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env);
***REMOVED***
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env){
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env);
***REMOVED***
public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env){
try { try {
System.out.println("Updating algorithm list: "+ file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream); svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n"); String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
***REMOVED***check if the algorithm is already in the list (match the class name)
for(String l: lines){
if(l.contains(algorithm.getClazz())){
return;
***REMOVED***
***REMOVED***
***REMOVED***the algorithm is not in the list. Add it
List<String> newContent = new LinkedList<>(Arrays.asList(lines));
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, user, env));
Collections.sort(newContent);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
byte[] originalContents = byteArrayOutputStream.toByteArray();
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : newContent) {
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file,md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
commitEditor.closeEdit();
***REMOVED***catch(Exception ex){
ex.printStackTrace();
***REMOVED***
finally {
svnRepository.closeSession();
***REMOVED*** ***REMOVED***
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String user, String env){
StringBuffer sb = new StringBuffer("| ");
sb.append(algorithm.getName() +" | ");
sb.append(user + " | ");
sb.append(algorithm.getCategory() + " | ");
sb.append(env + " | ");
sb.append("<notextile>./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
sb.append("none |");
return sb.toString();
***REMOVED***
public void updateSVN(String file, Collection<Dependency> deps) {
try {
System.out.println("Updating dependencies list: "+ file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> ldep = new LinkedList<>();
for(Dependency d: deps){
ldep.add(d.getName());
***REMOVED***
List<String> aa = this.checkMatch(lines, ldep); List<String> aa = this.checkMatch(lines, ldep);
Collections.sort(aa); Collections.sort(aa);

91
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/ServiceConfiguration.java Normal file
View File

@ -0,0 +1,91 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
***REMOVED***
import java.util.Properties;
public class ServiceConfiguration {
public static String home = System.getProperty("user.home");
private Properties props;
public ServiceConfiguration() {
this(home+"/dataminer-pool-manager/dpmConfig/service.properties");
***REMOVED***
public ServiceConfiguration(String configFile) {
this.props = new Properties();
FileInputStream input;
try {
input = new FileInputStream(configFile);
***REMOVED*** loading properites from properties file
try {
props.load(input);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (FileNotFoundException e1) {
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
***REMOVED***
public String getSVNRepository(){
return props.getProperty("svn.repository");
***REMOVED***
public String getSVNRProtoAlgorithmsList(){
return props.getProperty("svn.rproto.algorithms-list");
***REMOVED***
public String getSVNRProtoOSDepsList(){
return props.getProperty("svn.rproto.deps-os");
***REMOVED***
public String getSVNRProtoCRANDepsList(){
return props.getProperty("svn.rproto.deps-cran");
***REMOVED***
public String getSVNRProtoGitHubDepsList(){
return props.getProperty("svn.rproto.deps-github");
***REMOVED***
public String getSVNProdAlgorithmsList(){
return props.getProperty("svn.prod.algorithms-list");
***REMOVED***
public String getSVNProdOSDepsList(){
return props.getProperty("svn.prod.deps-os");
***REMOVED***
public String getSVNRProdCRANDepsList(){
return props.getProperty("svn.prod.deps-cran");
***REMOVED***
public String getSVNRProdGitHubDepsList(){
return props.getProperty("svn.prod.deps-github");
***REMOVED***
public String getCSVUrl() {
return props.getProperty("HAPROXY_CSV");
***REMOVED***
public String getStagingHost() {
return props.getProperty("STAGING_HOST");
***REMOVED***
public static void main(String[] args) throws FileNotFoundException {
ServiceConfiguration a = new ServiceConfiguration();
System.out.println(a.getStagingHost());
System.out.println(a.getCSVUrl());
***REMOVED***
***REMOVED***

14
src/main/resources/service.properties
View File

@ -3,3 +3,17 @@
STAGING_HOST: dataminer1-devnext.d4science.org STAGING_HOST: dataminer1-devnext.d4science.org
SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
#HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0 #HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0
svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube
svn.rproto.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/proto/algorithms
svn.rproto.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_deb_pkgs.txt
svn.rproto.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_cran_pkgs.txt
svn.rproto.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_github_pkgs.txt
svn.prod.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/prod/algorithms
svn.prod.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt
svn.prod.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
svn.prod.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt

2
src/main/webapp/WEB-INF/gcube-app.xml
View File

@ -10,4 +10,6 @@
<!-- <exclude>*</exclude> --> <!-- <exclude>*</exclude> -->
<exclude>/api/swagger.*</exclude>
</application> </application>

22
src/main/webapp/WEB-INF/web.xml
View File

@ -11,14 +11,32 @@
<servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class> <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
<init-param> <init-param>
<param-name>jersey.config.server.provider.packages</param-name> <param-name>jersey.config.server.provider.packages</param-name>
<param-value>org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value> <param-value>
io.swagger.jaxrs.listing,
org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
</init-param> </init-param>
<load-on-startup>1</load-on-startup> <load-on-startup>1</load-on-startup>
</servlet> </servlet>
<servlet-mapping> <servlet-mapping>
<servlet-name>REST-API</servlet-name> <servlet-name>REST-API</servlet-name>
<url-pattern>/rest/*</url-pattern> <url-pattern>/api/*</url-pattern>
</servlet-mapping> </servlet-mapping>
<!--
<servlet>
<servlet-name>Jersey2Config</servlet-name>
<servlet-class>io.swagger.jersey.config.JerseyJaxrsConfig</servlet-class>
<init-param>
<param-name>api.version</param-name>
<param-value>1.0.0</param-value>
</init-param>
<init-param>
<param-name>swagger.api.basepath</param-name>
<param-value>http:***REMOVED***localhost:8080/api</param-value>
</init-param>
<load-on-startup>2</load-on-startup>
</servlet>
-->
</web-app> </web-app>

5
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/JobTest.java
View File

@ -11,6 +11,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.service.StagingJob;
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder; import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater; import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
import org.tmatesoft.svn.core.SVNException; import org.tmatesoft.svn.core.SVNException;
public class JobTest { public class JobTest {
@ -19,13 +20,13 @@ public class JobTest {
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab"); ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/"); SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration());
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL"); Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
***REMOVED***test phase ***REMOVED***test phase
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster(); Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
Cluster rProtoCluster = ClusterBuilder.getRProtoCluster(); Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster); DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get());
job.start(); job.start();
***REMOVED***release phase ***REMOVED***release phase

26
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNTests.java Normal file
View File

@ -0,0 +1,26 @@
package org.gcube.dataanalysis.dataminerpoolmanager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
import org.tmatesoft.svn.core.SVNException;
***REMOVED***
/**
* Created by ggiammat on 5/17/17.
*/
public class SVNTests {
public static void main(String[] args) throws SVNException, IOException, InterruptedException {
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ggiammat/tmp/dmpm.properties"));
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
svnUpdater.updateRPRotoDeps(algo);
***REMOVED***
***REMOVED***

25
src/test/java/org/gcube/dataanalysis/dataminerpoolmanager/SVNUpdaterTest.java
View File

@ -1,25 +0,0 @@
package org.gcube.dataanalysis.dataminerpoolmanager;
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.tmatesoft.svn.core.SVNException;
public class SVNUpdaterTest {
public static void main(String[] args) throws SVNException, IOException {
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
String test = "testDep";
List<String> ldep = new LinkedList<>();
ldep.add(test);
svnUpdater.updateSVN("test_r_cran_pkgs.txt", ldep);
***REMOVED***
***REMOVED***