updated SVNUpdater

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148811 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gabriele Giammatteo 2017-05-17 16:16:25 +00:00
parent 8a5693f6b2
commit 04c9b93715
16 changed files with 491 additions and 308 deletions

272
pom.xml
View File

@ -1,141 +1,169 @@
<project xmlns="http:***REMOVED***maven.apache.org/POM/4.0.0" xmlns:xsi="http:***REMOVED***www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
xsi:schemaLocation="http:***REMOVED***maven.apache.org/POM/4.0.0 http:***REMOVED***maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<artifactId>maven-parent</artifactId>
<groupId>org.gcube.tools</groupId>
<version>1.0.0</version>
<relativePath />
</parent>
<parent>
<artifactId>maven-parent</artifactId>
<groupId>org.gcube.tools</groupId>
<version>1.0.0</version>
<relativePath/>
</parent>
<groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<version>2.0.0-SNAPSHOT</version>
<packaging>war</packaging>
<groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<version>2.0.0-SNAPSHOT</version>
<packaging>war</packaging>
<name>dataminer-pool-manager</name>
<description>
</description>
<name>dataminer-pool-manager</name>
<description>
</description>
<properties>
<distroDirectory>distro</distroDirectory>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
</properties>
<properties>
<distroDirectory>distro</distroDirectory>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<version.swagger>1.5.13</version.swagger>
<version.jersey>2.25.1</version.jersey>
</properties>
<dependencies>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version> -->
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>[2.5.0,2.6.0)</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>stringtemplate</artifactId>
<version>[4.0.0, 4.1.0)</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.5</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<dependencies>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
<artifactId>ic-client</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>2.23.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>2.22.1</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>2.22.1</version>
</dependency>
<dependency>
<groupId>org.gcube.core</groupId>
<artifactId>common-smartgears</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.gcube.resources.discovery</groupId>
<artifactId>ic-client</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-smartgears-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>[2.5.0,2.6.0)</version>
</dependency>
<dependency>
<groupId>org.antlr</groupId>
<artifactId>stringtemplate</artifactId>
<version>[4.0.0, 4.1.0)</version>
</dependency>
<dependency>
<groupId>org.tmatesoft.svnkit</groupId>
<artifactId>svnkit</artifactId>
<version>1.8.5</version>
</dependency>
<dependency>
<groupId>commons-configuration</groupId>
<artifactId>commons-configuration</artifactId>
<version>1.10</version>
</dependency>
<build>
<plugins>
<dependency>
<groupId>net.sf.opencsv</groupId>
<artifactId>opencsv</artifactId>
<version>2.3</version>
</dependency>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.16</version>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
<artifactId>jersey-media-json-jackson</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>${distroDirectory***REMOVED***/descriptor.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>servicearchive</id>
<phase>install</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
<dependency>
<groupId>org.glassfish.jersey.core</groupId>
<artifactId>jersey-client</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
</plugins>
</build>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet</artifactId>
<version>${version.jersey***REMOVED***</version>
</dependency>
<!--
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-jersey2-jaxrs</artifactId>
<version>${version.swagger***REMOVED***</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.6.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>io.swagger</groupId>
<artifactId>swagger-annotations</artifactId>
<version>${version.swagger***REMOVED***</version>
</dependency>
-->
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.gcube.distribution</groupId>
<artifactId>maven-smartgears-bom</artifactId>
<version>LATEST</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptors>
<descriptor>${distroDirectory***REMOVED***/descriptor.xml</descriptor>
</descriptors>
</configuration>
<executions>
<execution>
<id>servicearchive</id>
<phase>install</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -1,6 +1,8 @@
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.Vector;
@ -76,6 +78,36 @@ public class Algorithm {
public Collection<Dependency> getDependencies() {
return dependencies;
***REMOVED***
public Collection<Dependency> getOSDependencies(){
Set<Dependency> deps = new HashSet<>();
for(Dependency d: this.getDependencies()){
if(d.getType().equals("os")){
deps.add(d);
***REMOVED***
***REMOVED***
return deps;
***REMOVED***
public Collection<Dependency> getCranDependencies(){
Set<Dependency> deps = new HashSet<>();
for(Dependency d: this.getDependencies()){
if(d.getType().equals("cran")){
deps.add(d);
***REMOVED***
***REMOVED***
return deps;
***REMOVED***
public Collection<Dependency> getGitHubDependencies(){
Set<Dependency> deps = new HashSet<>();
for(Dependency d: this.getDependencies()){
if(d.getType().equals("github")){
deps.add(d);
***REMOVED***
***REMOVED***
return deps;
***REMOVED***
public void setDependencies(Collection<Dependency> deps) {
this.dependencies = deps;

View File

@ -74,27 +74,27 @@ public abstract class DMPMJob {
PrintStream ps = new PrintStream(fos);
return worker.execute(ps);
***REMOVED***
protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
for (Dependency d : algo.getDependencies()) {
if (d.getType().equals("os")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("cran")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
***REMOVED***
if (d.getType().equals("github")) {
List<String> ls = new LinkedList<String>();
ls.add(d.getName());
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException {
***REMOVED*** for (Dependency d : algo.getDependencies()) {
***REMOVED***
***REMOVED*** if (d.getType().equals("os")) {
***REMOVED*** List<String> ls = new LinkedList<String>();
***REMOVED*** ls.add(d.getName());
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
***REMOVED*** if (d.getType().equals("cran")) {
***REMOVED*** List<String> ls = new LinkedList<String>();
***REMOVED*** ls.add(d.getName());
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
***REMOVED*** if (d.getType().equals("github")) {
***REMOVED*** List<String> ls = new LinkedList<String>();
***REMOVED*** ls.add(d.getName());
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
***REMOVED*** ***REMOVED***
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
***REMOVED***

View File

@ -5,10 +5,11 @@ import java.io.FileNotFoundException;
***REMOVED***
import java.util.Scanner;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.Props;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.tmatesoft.svn.core.SVNException;
@ -20,7 +21,7 @@ public class DataminerPoolManager {
public DataminerPoolManager() {
try {
***REMOVED***TODO: read this from configuration
this.svnUpdater = new SVNUpdater(new Props().getSVNrepo());
this.svnUpdater = new SVNUpdater(new ServiceConfiguration());
***REMOVED*** catch (SVNException e) {
e.printStackTrace();
***REMOVED***
@ -33,7 +34,7 @@ public class DataminerPoolManager {
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster);
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get());
String id = job.start();
return id;
***REMOVED***
@ -42,7 +43,7 @@ public class DataminerPoolManager {
Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster);
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE);
String id = job.start();
return id;
***REMOVED***

View File

@ -8,20 +8,24 @@ public class ProductionPublishingJob extends DMPMJob {
private Algorithm algorithm;
private Cluster prodCluster;
private String targetVREName;
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster) {
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName) {
super(svnUpdater);
this.algorithm = algorithm;
this.prodCluster = prodCluster;
this.targetVREName = targetVREName;
***REMOVED***
@Override
protected void execute() {
try {
this.updateSVNDependencies(this.algorithm, false);
this.svnUpdater.updateProdDeps(this.algorithm);
int ret = this.executeAnsibleWorker(
createWorker(this.algorithm, this.prodCluster, false, "gcube"));
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
***REMOVED*** int ret = this.executeAnsibleWorker(
***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube"));
***REMOVED*** catch (Exception e) {
e.printStackTrace();

View File

@ -10,12 +10,19 @@ public class StagingJob extends DMPMJob {
private Algorithm algorithm;
private Cluster stagingCluster;
private Cluster rProtoCluster;
private String rProtoVREName;
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster) {
public StagingJob(
SVNUpdater svnUpdater,
Algorithm algorithm,
Cluster stagingCluster,
Cluster rProtoCluster,
String rProtoVREName) {
super(svnUpdater);
this.algorithm = algorithm;
this.stagingCluster = stagingCluster;
this.rProtoCluster = rProtoCluster;
this.rProtoVREName = rProtoVREName;
***REMOVED***
@Override
@ -26,13 +33,13 @@ public class StagingJob extends DMPMJob {
createWorker(this.algorithm, this.stagingCluster, true, "root"));
if(ret == 0){
this.svnUpdater.updateRPRotoDeps(this.algorithm);
this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, "DataMiner Pool Manager", "Proto");
this.updateSVNDependencies(this.algorithm, true);
int ret2 = this.executeAnsibleWorker(
createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
***REMOVED*** int ret2 = this.executeAnsibleWorker(
***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
***REMOVED***

View File

@ -15,7 +15,7 @@ public class ClusterBuilder {
***REMOVED***1. to complete
public static Cluster getStagingDataminerCluster() throws FileNotFoundException{
Cluster cluster = new Cluster();
Props p = new Props();
ServiceConfiguration p = new ServiceConfiguration();
Host h = new Host(p.getStagingHost());
***REMOVED***TODO: read this from configuration or IS?
h.setName(p.getStagingHost());

View File

@ -1,103 +0,0 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
***REMOVED***
import java.io.InputStream;
import java.util.Properties;
public class Props {
public static String home = System.getProperty("user.home");
public Props() {
***REMOVED***
public String getCSVUrl() {
Properties props = new Properties();
***REMOVED***read from war
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
***REMOVED***read from filesystem
FileInputStream input;
try {
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
***REMOVED*** loading properites from properties file
try {
props.load(input);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (FileNotFoundException e1) {
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
***REMOVED*** reading propeprty
String path = props.getProperty("HAPROXY_CSV");
return path;
***REMOVED***
public String getSVNrepo() {
Properties props = new Properties();
***REMOVED***read from war
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
***REMOVED***read from filesystem
FileInputStream input;
try {
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
***REMOVED*** loading properites from properties file
try {
props.load(input);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (FileNotFoundException e1) {
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
***REMOVED*** reading propeprty
String path = props.getProperty("SVN_REPO");
return path;
***REMOVED***
public String getStagingHost() {
Properties props = new Properties();
***REMOVED***read from war
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
***REMOVED***read from filesystem
FileInputStream input;
try {
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
***REMOVED*** loading properites from properties file
try {
props.load(input);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (FileNotFoundException e1) {
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
***REMOVED*** reading propeprty
String path = props.getProperty("STAGING_HOST");
return path;
***REMOVED***
public static void main(String[] args) throws FileNotFoundException {
Props a = new Props();
System.out.println(new Props().getSVNrepo());
System.out.println(a.getStagingHost());
System.out.println(a.getCSVUrl());
***REMOVED***
***REMOVED***

View File

@ -1,5 +1,7 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
@ -21,32 +23,117 @@ import java.util.*;
*/
public class SVNUpdater {
private SVNRepository svnRepository;
private ServiceConfiguration configuration;
public SVNUpdater(String rootURL) throws SVNException {
this.svnRepository = this.getSvnRepository(
rootURL);
***REMOVED***
private SVNRepository getSvnRepository(String url) throws SVNException {
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
public SVNUpdater(ServiceConfiguration configuration) throws SVNException {
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
repository.setAuthenticationManager(authManager);
System.out.println(repository.getLocation());
return repository;
this.svnRepository.setAuthenticationManager(authManager);
this.configuration = configuration;
***REMOVED***
public void updateRPRotoDeps(Algorithm algorithm){
this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies());
this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies());
this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
***REMOVED***
public void updateSVN(String file, List<String> ldep) {
public void updateProdDeps(Algorithm algorithm){
this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
***REMOVED***
public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env){
this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env);
***REMOVED***
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env){
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env);
***REMOVED***
public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env){
try {
System.out.println("Updating algorithm list: "+ file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
***REMOVED***check if the algorithm is already in the list (match the class name)
for(String l: lines){
if(l.contains(algorithm.getClazz())){
return;
***REMOVED***
***REMOVED***
***REMOVED***the algorithm is not in the list. Add it
List<String> newContent = new LinkedList<>(Arrays.asList(lines));
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, user, env));
Collections.sort(newContent);
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
byte[] originalContents = byteArrayOutputStream.toByteArray();
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
commitEditor.openRoot(-1);
commitEditor.openFile(file, -1);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
for (String line : newContent) {
baos.write(line.getBytes());
baos.write("\n".getBytes());
***REMOVED***
byte[] bytes = baos.toByteArray();
commitEditor.applyTextDelta(file,md5(originalContents));
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
new ByteArrayInputStream(bytes), commitEditor, true);
commitEditor.closeFile(file, checksum);
commitEditor.closeEdit();
***REMOVED***catch(Exception ex){
ex.printStackTrace();
***REMOVED***
finally {
svnRepository.closeSession();
***REMOVED*** ***REMOVED***
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String user, String env){
StringBuffer sb = new StringBuffer("| ");
sb.append(algorithm.getName() +" | ");
sb.append(user + " | ");
sb.append(algorithm.getCategory() + " | ");
sb.append(env + " | ");
sb.append("<notextile>./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
sb.append("none |");
return sb.toString();
***REMOVED***
public void updateSVN(String file, Collection<Dependency> deps) {
try {
System.out.println("Updating dependencies list: "+ file);
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
List<String> ldep = new LinkedList<>();
for(Dependency d: deps){
ldep.add(d.getName());
***REMOVED***
List<String> aa = this.checkMatch(lines, ldep);
Collections.sort(aa);

View File

@ -0,0 +1,91 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
***REMOVED***
import java.util.Properties;
public class ServiceConfiguration {
public static String home = System.getProperty("user.home");
private Properties props;
public ServiceConfiguration() {
this(home+"/dataminer-pool-manager/dpmConfig/service.properties");
***REMOVED***
public ServiceConfiguration(String configFile) {
this.props = new Properties();
FileInputStream input;
try {
input = new FileInputStream(configFile);
***REMOVED*** loading properites from properties file
try {
props.load(input);
***REMOVED*** catch (IOException e) {
***REMOVED*** TODO Auto-generated catch block
e.printStackTrace();
***REMOVED***
***REMOVED*** catch (FileNotFoundException e1) {
***REMOVED*** TODO Auto-generated catch block
e1.printStackTrace();
***REMOVED***
***REMOVED***
public String getSVNRepository(){
return props.getProperty("svn.repository");
***REMOVED***
public String getSVNRProtoAlgorithmsList(){
return props.getProperty("svn.rproto.algorithms-list");
***REMOVED***
public String getSVNRProtoOSDepsList(){
return props.getProperty("svn.rproto.deps-os");
***REMOVED***
public String getSVNRProtoCRANDepsList(){
return props.getProperty("svn.rproto.deps-cran");
***REMOVED***
public String getSVNRProtoGitHubDepsList(){
return props.getProperty("svn.rproto.deps-github");
***REMOVED***
public String getSVNProdAlgorithmsList(){
return props.getProperty("svn.prod.algorithms-list");
***REMOVED***
public String getSVNProdOSDepsList(){
return props.getProperty("svn.prod.deps-os");
***REMOVED***
public String getSVNRProdCRANDepsList(){
return props.getProperty("svn.prod.deps-cran");
***REMOVED***
public String getSVNRProdGitHubDepsList(){
return props.getProperty("svn.prod.deps-github");
***REMOVED***
public String getCSVUrl() {
return props.getProperty("HAPROXY_CSV");
***REMOVED***
public String getStagingHost() {
return props.getProperty("STAGING_HOST");
***REMOVED***
public static void main(String[] args) throws FileNotFoundException {
ServiceConfiguration a = new ServiceConfiguration();
System.out.println(a.getStagingHost());
System.out.println(a.getCSVUrl());
***REMOVED***
***REMOVED***

View File

@ -3,3 +3,17 @@
STAGING_HOST: dataminer1-devnext.d4science.org
SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
#HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0
svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube
svn.rproto.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/proto/algorithms
svn.rproto.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_deb_pkgs.txt
svn.rproto.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_cran_pkgs.txt
svn.rproto.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_github_pkgs.txt
svn.prod.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/prod/algorithms
svn.prod.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt
svn.prod.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
svn.prod.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt

View File

@ -9,5 +9,7 @@
<!--<persistence location="/some/custom/location</persistence" />-->
<!-- <exclude>*</exclude> -->
<exclude>/api/swagger.*</exclude>
</application>

View File

@ -11,14 +11,32 @@
<servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
<init-param>
<param-name>jersey.config.server.provider.packages</param-name>
<param-value>org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
<param-value>
io.swagger.jaxrs.listing,
org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
</init-param>
<load-on-startup>1</load-on-startup>
</servlet>
<servlet-mapping>
<servlet-name>REST-API</servlet-name>
<url-pattern>/rest/*</url-pattern>
<url-pattern>/api/*</url-pattern>
</servlet-mapping>
<!--
<servlet>
<servlet-name>Jersey2Config</servlet-name>
<servlet-class>io.swagger.jersey.config.JerseyJaxrsConfig</servlet-class>
<init-param>
<param-name>api.version</param-name>
<param-value>1.0.0</param-value>
</init-param>
<init-param>
<param-name>swagger.api.basepath</param-name>
<param-value>http:***REMOVED***localhost:8080/api</param-value>
</init-param>
<load-on-startup>2</load-on-startup>
</servlet>
-->
</web-app>

View File

@ -11,6 +11,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.service.StagingJob;
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
import org.tmatesoft.svn.core.SVNException;
public class JobTest {
@ -19,13 +20,13 @@ public class JobTest {
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration());
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
***REMOVED***test phase
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster);
DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get());
job.start();
***REMOVED***release phase

View File

@ -0,0 +1,26 @@
package org.gcube.dataanalysis.dataminerpoolmanager;
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Algorithm;
import org.gcube.dataanalysis.dataminer.poolmanager.util.AlgorithmBuilder;
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
import org.tmatesoft.svn.core.SVNException;
***REMOVED***
/**
* Created by ggiammat on 5/17/17.
*/
public class SVNTests {
public static void main(String[] args) throws SVNException, IOException, InterruptedException {
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ggiammat/tmp/dmpm.properties"));
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
svnUpdater.updateRPRotoDeps(algo);
***REMOVED***
***REMOVED***

View File

@ -1,25 +0,0 @@
package org.gcube.dataanalysis.dataminerpoolmanager;
***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED***
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
import org.tmatesoft.svn.core.SVNException;
public class SVNUpdaterTest {
public static void main(String[] args) throws SVNException, IOException {
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
String test = "testDep";
List<String> ldep = new LinkedList<>();
ldep.add(test);
svnUpdater.updateSVN("test_r_cran_pkgs.txt", ldep);
***REMOVED***
***REMOVED***