updated SVNUpdater
git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@148811 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
Gabriele Giammatteo
parent
c9b04e7d89
commit
0bc11c4cee
64
pom.xml
64
pom.xml
|
|
@ -21,15 +21,28 @@
|
|||
<properties>
|
||||
<distroDirectory>distro</distroDirectory>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<version.swagger>1.5.13</version.swagger>
|
||||
<version.jersey>2.25.1</version.jersey>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.core</groupId>
|
||||
<artifactId>common-smartgears</artifactId>
|
||||
<!-- <version>[1.0.0-SNAPSHOT,1.2.7)</version> -->
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.gcube.resources.discovery</groupId>
|
||||
<artifactId>ic-client</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
|
||||
|
||||
|
||||
<dependency>
|
||||
<groupId>commons-io</groupId>
|
||||
<artifactId>commons-io</artifactId>
|
||||
|
|
@ -56,11 +69,7 @@
|
|||
<artifactId>opencsv</artifactId>
|
||||
<version>2.3</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.gcube.resources.discovery</groupId>
|
||||
<artifactId>ic-client</artifactId>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.yaml</groupId>
|
||||
<artifactId>snakeyaml</artifactId>
|
||||
|
|
@ -69,20 +78,48 @@
|
|||
<dependency>
|
||||
<groupId>org.glassfish.jersey.media</groupId>
|
||||
<artifactId>jersey-media-json-jackson</artifactId>
|
||||
<version>2.23.1</version>
|
||||
<version>$***REMOVED***version.jersey***REMOVED***</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.core</groupId>
|
||||
<artifactId>jersey-client</artifactId>
|
||||
<version>2.22.1</version>
|
||||
<version>$***REMOVED***version.jersey***REMOVED***</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.glassfish.jersey.containers</groupId>
|
||||
<artifactId>jersey-container-servlet</artifactId>
|
||||
<version>2.22.1</version>
|
||||
<version>$***REMOVED***version.jersey***REMOVED***</version>
|
||||
</dependency>
|
||||
|
||||
<!--
|
||||
<dependency>
|
||||
<groupId>io.swagger</groupId>
|
||||
<artifactId>swagger-jersey2-jaxrs</artifactId>
|
||||
<version>$***REMOVED***version.swagger***REMOVED***</version>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-databind</artifactId>
|
||||
<version>2.6.0</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>com.fasterxml.jackson.core</groupId>
|
||||
<artifactId>jackson-core</artifactId>
|
||||
<version>2.6.0</version>
|
||||
<scope>provided</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>io.swagger</groupId>
|
||||
<artifactId>swagger-annotations</artifactId>
|
||||
<version>$***REMOVED***version.swagger***REMOVED***</version>
|
||||
</dependency>
|
||||
-->
|
||||
<dependency>
|
||||
<groupId>org.slf4j</groupId>
|
||||
<artifactId>slf4j-api</artifactId>
|
||||
|
|
@ -90,7 +127,6 @@
|
|||
</dependency>
|
||||
|
||||
|
||||
|
||||
</dependencies>
|
||||
|
||||
<dependencyManagement>
|
||||
|
|
@ -108,14 +144,6 @@
|
|||
<build>
|
||||
<plugins>
|
||||
|
||||
<plugin>
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<configuration>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
</configuration>
|
||||
</plugin>
|
||||
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-assembly-plugin</artifactId>
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.datamodel;
|
||||
|
||||
import java.util.Collection;
|
||||
import java.util.HashSet;
|
||||
***REMOVED***
|
||||
import java.util.Vector;
|
||||
|
||||
|
||||
|
|
@ -77,6 +79,36 @@ public class Algorithm ***REMOVED***
|
|||
return dependencies;
|
||||
***REMOVED***
|
||||
|
||||
public Collection<Dependency> getOSDependencies()***REMOVED***
|
||||
Set<Dependency> deps = new HashSet<>();
|
||||
for(Dependency d: this.getDependencies())***REMOVED***
|
||||
if(d.getType().equals("os"))***REMOVED***
|
||||
deps.add(d);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return deps;
|
||||
***REMOVED***
|
||||
|
||||
public Collection<Dependency> getCranDependencies()***REMOVED***
|
||||
Set<Dependency> deps = new HashSet<>();
|
||||
for(Dependency d: this.getDependencies())***REMOVED***
|
||||
if(d.getType().equals("cran"))***REMOVED***
|
||||
deps.add(d);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return deps;
|
||||
***REMOVED***
|
||||
|
||||
public Collection<Dependency> getGitHubDependencies()***REMOVED***
|
||||
Set<Dependency> deps = new HashSet<>();
|
||||
for(Dependency d: this.getDependencies())***REMOVED***
|
||||
if(d.getType().equals("github"))***REMOVED***
|
||||
deps.add(d);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
return deps;
|
||||
***REMOVED***
|
||||
|
||||
public void setDependencies(Collection<Dependency> deps) ***REMOVED***
|
||||
this.dependencies = deps;
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -75,26 +75,26 @@ public abstract class DMPMJob ***REMOVED***
|
|||
return worker.execute(ps);
|
||||
***REMOVED***
|
||||
|
||||
|
||||
protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED***
|
||||
for (Dependency d : algo.getDependencies()) ***REMOVED***
|
||||
|
||||
if (d.getType().equals("os")) ***REMOVED***
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("cran")) ***REMOVED***
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
if (d.getType().equals("github")) ***REMOVED***
|
||||
List<String> ls = new LinkedList<String>();
|
||||
ls.add(d.getName());
|
||||
this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** protected void updateSVNDependencies(Algorithm algo, boolean stagingVRE) throws IOException, SVNException ***REMOVED***
|
||||
***REMOVED*** for (Dependency d : algo.getDependencies()) ***REMOVED***
|
||||
***REMOVED***
|
||||
***REMOVED*** if (d.getType().equals("os")) ***REMOVED***
|
||||
***REMOVED*** List<String> ls = new LinkedList<String>();
|
||||
***REMOVED*** ls.add(d.getName());
|
||||
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_deb_pkgs.txt", ls);
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** if (d.getType().equals("cran")) ***REMOVED***
|
||||
***REMOVED*** List<String> ls = new LinkedList<String>();
|
||||
***REMOVED*** ls.add(d.getName());
|
||||
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_cran_pkgs.txt", ls);
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** if (d.getType().equals("github")) ***REMOVED***
|
||||
***REMOVED*** List<String> ls = new LinkedList<String>();
|
||||
***REMOVED*** ls.add(d.getName());
|
||||
***REMOVED*** this.svnUpdater.updateSVN((stagingVRE ? "test_": "") + "r_github_pkgs.txt", ls);
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED*** ***REMOVED***
|
||||
***REMOVED******REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -5,10 +5,11 @@ import java.io.FileNotFoundException;
|
|||
***REMOVED***
|
||||
import java.util.Scanner;
|
||||
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Cluster;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.Props;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
***REMOVED***
|
||||
|
||||
|
|
@ -20,7 +21,7 @@ public class DataminerPoolManager ***REMOVED***
|
|||
public DataminerPoolManager() ***REMOVED***
|
||||
try ***REMOVED***
|
||||
***REMOVED***TODO: read this from configuration
|
||||
this.svnUpdater = new SVNUpdater(new Props().getSVNrepo());
|
||||
this.svnUpdater = new SVNUpdater(new ServiceConfiguration());
|
||||
***REMOVED*** catch (SVNException e) ***REMOVED***
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
|
|
@ -33,7 +34,7 @@ public class DataminerPoolManager ***REMOVED***
|
|||
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
|
||||
Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
|
||||
|
||||
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster);
|
||||
DMPMJob job = new StagingJob(this.svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get());
|
||||
String id = job.start();
|
||||
return id;
|
||||
***REMOVED***
|
||||
|
|
@ -42,7 +43,7 @@ public class DataminerPoolManager ***REMOVED***
|
|||
|
||||
Cluster prodCluster = ClusterBuilder.getVRECluster(targetVREToken, targetVRE);
|
||||
|
||||
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster);
|
||||
DMPMJob job = new ProductionPublishingJob(this.svnUpdater, algo, prodCluster, targetVRE);
|
||||
String id = job.start();
|
||||
return id;
|
||||
***REMOVED***
|
||||
|
|
|
|||
|
|
@ -8,20 +8,24 @@ public class ProductionPublishingJob extends DMPMJob ***REMOVED***
|
|||
|
||||
private Algorithm algorithm;
|
||||
private Cluster prodCluster;
|
||||
private String targetVREName;
|
||||
|
||||
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster) ***REMOVED***
|
||||
public ProductionPublishingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster prodCluster, String targetVREName) ***REMOVED***
|
||||
super(svnUpdater);
|
||||
this.algorithm = algorithm;
|
||||
this.prodCluster = prodCluster;
|
||||
this.targetVREName = targetVREName;
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
protected void execute() ***REMOVED***
|
||||
try ***REMOVED***
|
||||
this.updateSVNDependencies(this.algorithm, false);
|
||||
this.svnUpdater.updateProdDeps(this.algorithm);
|
||||
|
||||
int ret = this.executeAnsibleWorker(
|
||||
createWorker(this.algorithm, this.prodCluster, false, "gcube"));
|
||||
this.svnUpdater.updateSVNProdAlgorithmList(this.algorithm, this.targetVREName, "DataMiner Pool Manager", "Prod");
|
||||
|
||||
***REMOVED*** int ret = this.executeAnsibleWorker(
|
||||
***REMOVED*** createWorker(this.algorithm, this.prodCluster, false, "gcube"));
|
||||
|
||||
***REMOVED*** catch (Exception e) ***REMOVED***
|
||||
e.printStackTrace();
|
||||
|
|
|
|||
|
|
@ -10,12 +10,19 @@ public class StagingJob extends DMPMJob ***REMOVED***
|
|||
private Algorithm algorithm;
|
||||
private Cluster stagingCluster;
|
||||
private Cluster rProtoCluster;
|
||||
private String rProtoVREName;
|
||||
|
||||
public StagingJob(SVNUpdater svnUpdater, Algorithm algorithm, Cluster stagingCluster, Cluster rProtoCluster) ***REMOVED***
|
||||
public StagingJob(
|
||||
SVNUpdater svnUpdater,
|
||||
Algorithm algorithm,
|
||||
Cluster stagingCluster,
|
||||
Cluster rProtoCluster,
|
||||
String rProtoVREName) ***REMOVED***
|
||||
super(svnUpdater);
|
||||
this.algorithm = algorithm;
|
||||
this.stagingCluster = stagingCluster;
|
||||
this.rProtoCluster = rProtoCluster;
|
||||
this.rProtoVREName = rProtoVREName;
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
|
@ -27,12 +34,12 @@ public class StagingJob extends DMPMJob ***REMOVED***
|
|||
|
||||
if(ret == 0)***REMOVED***
|
||||
|
||||
this.svnUpdater.updateRPRotoDeps(this.algorithm);
|
||||
|
||||
this.updateSVNDependencies(this.algorithm, true);
|
||||
this.svnUpdater.updateSVNRProtoAlgorithmList(this.algorithm, this.rProtoVREName, "DataMiner Pool Manager", "Proto");
|
||||
|
||||
|
||||
int ret2 = this.executeAnsibleWorker(
|
||||
createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
|
||||
***REMOVED*** int ret2 = this.executeAnsibleWorker(
|
||||
***REMOVED*** createWorker(this.algorithm, this.rProtoCluster, false, "gcube"));
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ public class ClusterBuilder ***REMOVED***
|
|||
***REMOVED***1. to complete
|
||||
public static Cluster getStagingDataminerCluster() throws FileNotFoundException***REMOVED***
|
||||
Cluster cluster = new Cluster();
|
||||
Props p = new Props();
|
||||
ServiceConfiguration p = new ServiceConfiguration();
|
||||
Host h = new Host(p.getStagingHost());
|
||||
***REMOVED***TODO: read this from configuration or IS?
|
||||
h.setName(p.getStagingHost());
|
||||
|
|
|
|||
|
|
@ -1,103 +0,0 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.util;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
***REMOVED***
|
||||
import java.io.InputStream;
|
||||
import java.util.Properties;
|
||||
|
||||
public class Props ***REMOVED***
|
||||
public static String home = System.getProperty("user.home");
|
||||
|
||||
public Props() ***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
public String getCSVUrl() ***REMOVED***
|
||||
Properties props = new Properties();
|
||||
***REMOVED***read from war
|
||||
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
|
||||
|
||||
***REMOVED***read from filesystem
|
||||
FileInputStream input;
|
||||
try ***REMOVED***
|
||||
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
|
||||
***REMOVED*** loading properites from properties file
|
||||
try ***REMOVED***
|
||||
props.load(input);
|
||||
***REMOVED*** catch (IOException e) ***REMOVED***
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e1.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED*** reading propeprty
|
||||
String path = props.getProperty("HAPROXY_CSV");
|
||||
return path;
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public String getSVNrepo() ***REMOVED***
|
||||
Properties props = new Properties();
|
||||
***REMOVED***read from war
|
||||
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
|
||||
|
||||
***REMOVED***read from filesystem
|
||||
FileInputStream input;
|
||||
try ***REMOVED***
|
||||
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
|
||||
***REMOVED*** loading properites from properties file
|
||||
try ***REMOVED***
|
||||
props.load(input);
|
||||
***REMOVED*** catch (IOException e) ***REMOVED***
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e1.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED*** reading propeprty
|
||||
String path = props.getProperty("SVN_REPO");
|
||||
return path;
|
||||
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public String getStagingHost() ***REMOVED***
|
||||
Properties props = new Properties();
|
||||
***REMOVED***read from war
|
||||
***REMOVED***java.io.InputStream input = Props.class.getClassLoader().getResourceAsStream("service.properties");
|
||||
|
||||
***REMOVED***read from filesystem
|
||||
FileInputStream input;
|
||||
try ***REMOVED***
|
||||
input = new FileInputStream(home+"/dataminer-pool-manager/dpmConfig/service.properties");
|
||||
***REMOVED*** loading properites from properties file
|
||||
try ***REMOVED***
|
||||
props.load(input);
|
||||
***REMOVED*** catch (IOException e) ***REMOVED***
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e1.printStackTrace();
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED*** reading propeprty
|
||||
String path = props.getProperty("STAGING_HOST");
|
||||
return path;
|
||||
|
||||
***REMOVED***
|
||||
|
||||
public static void main(String[] args) throws FileNotFoundException ***REMOVED***
|
||||
Props a = new Props();
|
||||
System.out.println(new Props().getSVNrepo());
|
||||
System.out.println(a.getStagingHost());
|
||||
System.out.println(a.getCSVUrl());
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.util;
|
||||
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.datamodel.Dependency;
|
||||
***REMOVED***
|
||||
import org.tmatesoft.svn.core.SVNURL;
|
||||
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
|
||||
|
|
@ -21,32 +23,117 @@ import java.util.*;
|
|||
*/
|
||||
public class SVNUpdater ***REMOVED***
|
||||
|
||||
|
||||
private SVNRepository svnRepository;
|
||||
private ServiceConfiguration configuration;
|
||||
|
||||
public SVNUpdater(String rootURL) throws SVNException ***REMOVED***
|
||||
this.svnRepository = this.getSvnRepository(
|
||||
rootURL);
|
||||
***REMOVED***
|
||||
|
||||
|
||||
private SVNRepository getSvnRepository(String url) throws SVNException ***REMOVED***
|
||||
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
||||
public SVNUpdater(ServiceConfiguration configuration) throws SVNException ***REMOVED***
|
||||
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
|
||||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
||||
repository.setAuthenticationManager(authManager);
|
||||
System.out.println(repository.getLocation());
|
||||
|
||||
return repository;
|
||||
this.svnRepository.setAuthenticationManager(authManager);
|
||||
this.configuration = configuration;
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public void updateRPRotoDeps(Algorithm algorithm)***REMOVED***
|
||||
this.updateSVN(this.configuration.getSVNRProtoOSDepsList(), algorithm.getOSDependencies());
|
||||
this.updateSVN(this.configuration.getSVNRProtoCRANDepsList(), algorithm.getCranDependencies());
|
||||
this.updateSVN(this.configuration.getSVNRProtoGitHubDepsList(), algorithm.getGitHubDependencies());
|
||||
***REMOVED***
|
||||
|
||||
public void updateSVN(String file, List<String> ldep) ***REMOVED***
|
||||
public void updateProdDeps(Algorithm algorithm)***REMOVED***
|
||||
this.updateSVN(this.configuration.getSVNProdOSDepsList(), algorithm.getOSDependencies());
|
||||
this.updateSVN(this.configuration.getSVNRProdCRANDepsList(), algorithm.getCranDependencies());
|
||||
this.updateSVN(this.configuration.getSVNRProdGitHubDepsList(), algorithm.getGitHubDependencies());
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public void updateSVNRProtoAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
|
||||
this.updateSVNAlgorithmList(this.configuration.getSVNRProtoAlgorithmsList(), algorithm, targetVRE, user, env);
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public void updateSVNProdAlgorithmList(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
|
||||
this.updateSVNAlgorithmList(this.configuration.getSVNProdAlgorithmsList(), algorithm, targetVRE, user, env);
|
||||
***REMOVED***
|
||||
|
||||
public void updateSVNAlgorithmList(String file, Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
|
||||
try ***REMOVED***
|
||||
System.out.println("Updating algorithm list: "+ file);
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
|
||||
***REMOVED***check if the algorithm is already in the list (match the class name)
|
||||
for(String l: lines)***REMOVED***
|
||||
if(l.contains(algorithm.getClazz()))***REMOVED***
|
||||
return;
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***the algorithm is not in the list. Add it
|
||||
List<String> newContent = new LinkedList<>(Arrays.asList(lines));
|
||||
newContent.add(this.generateAlgorithmEntry(algorithm, targetVRE, user, env));
|
||||
Collections.sort(newContent);
|
||||
|
||||
|
||||
final SVNDeltaGenerator deltaGenerator = new SVNDeltaGenerator();
|
||||
|
||||
byte[] originalContents = byteArrayOutputStream.toByteArray();
|
||||
|
||||
final ISVNEditor commitEditor = svnRepository.getCommitEditor("update dependencies", null);
|
||||
commitEditor.openRoot(-1);
|
||||
commitEditor.openFile(file, -1);
|
||||
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream();
|
||||
|
||||
for (String line : newContent) ***REMOVED***
|
||||
baos.write(line.getBytes());
|
||||
baos.write("\n".getBytes());
|
||||
***REMOVED***
|
||||
byte[] bytes = baos.toByteArray();
|
||||
|
||||
commitEditor.applyTextDelta(file,md5(originalContents));
|
||||
|
||||
final String checksum = deltaGenerator.sendDelta(file, new ByteArrayInputStream(originalContents), 0,
|
||||
new ByteArrayInputStream(bytes), commitEditor, true);
|
||||
commitEditor.closeFile(file, checksum);
|
||||
commitEditor.closeEdit();
|
||||
|
||||
|
||||
***REMOVED***catch(Exception ex)***REMOVED***
|
||||
ex.printStackTrace();
|
||||
***REMOVED***
|
||||
|
||||
finally ***REMOVED***
|
||||
svnRepository.closeSession();
|
||||
***REMOVED*** ***REMOVED***
|
||||
|
||||
|
||||
public String generateAlgorithmEntry(Algorithm algorithm, String targetVRE, String user, String env)***REMOVED***
|
||||
StringBuffer sb = new StringBuffer("| ");
|
||||
sb.append(algorithm.getName() +" | ");
|
||||
sb.append(user + " | ");
|
||||
sb.append(algorithm.getCategory() + " | ");
|
||||
sb.append(env + " | ");
|
||||
sb.append("<notextile>./addAlgorithm.sh "+ algorithm.getName()+" " + algorithm.getCategory() + " " + algorithm.getClazz() + " " + targetVRE + " " + algorithm.getAlgorithmType() + " N " + algorithm.getPackageURL() + " \"" + algorithm.getDescription() + "\" </notextile> | ");
|
||||
sb.append("none |");
|
||||
return sb.toString();
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public void updateSVN(String file, Collection<Dependency> deps) ***REMOVED***
|
||||
try ***REMOVED***
|
||||
System.out.println("Updating dependencies list: "+ file);
|
||||
|
||||
final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
|
||||
svnRepository.getFile(file, SVNRepository.INVALID_REVISION, null, byteArrayOutputStream);
|
||||
String lines[] = byteArrayOutputStream.toString().split("\\r?\\n");
|
||||
|
||||
List<String> ldep = new LinkedList<>();
|
||||
for(Dependency d: deps)***REMOVED***
|
||||
ldep.add(d.getName());
|
||||
***REMOVED***
|
||||
List<String> aa = this.checkMatch(lines, ldep);
|
||||
Collections.sort(aa);
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,91 @@
|
|||
package org.gcube.dataanalysis.dataminer.poolmanager.util;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
***REMOVED***
|
||||
import java.util.Properties;
|
||||
|
||||
public class ServiceConfiguration ***REMOVED***
|
||||
public static String home = System.getProperty("user.home");
|
||||
|
||||
private Properties props;
|
||||
|
||||
public ServiceConfiguration() ***REMOVED***
|
||||
this(home+"/dataminer-pool-manager/dpmConfig/service.properties");
|
||||
***REMOVED***
|
||||
|
||||
public ServiceConfiguration(String configFile) ***REMOVED***
|
||||
this.props = new Properties();
|
||||
|
||||
FileInputStream input;
|
||||
|
||||
try ***REMOVED***
|
||||
input = new FileInputStream(configFile);
|
||||
***REMOVED*** loading properites from properties file
|
||||
try ***REMOVED***
|
||||
props.load(input);
|
||||
***REMOVED*** catch (IOException e) ***REMOVED***
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED*** catch (FileNotFoundException e1) ***REMOVED***
|
||||
***REMOVED*** TODO Auto-generated catch block
|
||||
e1.printStackTrace();
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
||||
|
||||
|
||||
public String getSVNRepository()***REMOVED***
|
||||
return props.getProperty("svn.repository");
|
||||
***REMOVED***
|
||||
|
||||
public String getSVNRProtoAlgorithmsList()***REMOVED***
|
||||
return props.getProperty("svn.rproto.algorithms-list");
|
||||
***REMOVED***
|
||||
|
||||
public String getSVNRProtoOSDepsList()***REMOVED***
|
||||
return props.getProperty("svn.rproto.deps-os");
|
||||
***REMOVED***
|
||||
|
||||
public String getSVNRProtoCRANDepsList()***REMOVED***
|
||||
return props.getProperty("svn.rproto.deps-cran");
|
||||
***REMOVED***
|
||||
|
||||
public String getSVNRProtoGitHubDepsList()***REMOVED***
|
||||
return props.getProperty("svn.rproto.deps-github");
|
||||
***REMOVED***
|
||||
|
||||
public String getSVNProdAlgorithmsList()***REMOVED***
|
||||
return props.getProperty("svn.prod.algorithms-list");
|
||||
***REMOVED***
|
||||
|
||||
public String getSVNProdOSDepsList()***REMOVED***
|
||||
return props.getProperty("svn.prod.deps-os");
|
||||
***REMOVED***
|
||||
|
||||
public String getSVNRProdCRANDepsList()***REMOVED***
|
||||
return props.getProperty("svn.prod.deps-cran");
|
||||
***REMOVED***
|
||||
|
||||
public String getSVNRProdGitHubDepsList()***REMOVED***
|
||||
return props.getProperty("svn.prod.deps-github");
|
||||
***REMOVED***
|
||||
|
||||
public String getCSVUrl() ***REMOVED***
|
||||
return props.getProperty("HAPROXY_CSV");
|
||||
***REMOVED***
|
||||
|
||||
|
||||
public String getStagingHost() ***REMOVED***
|
||||
return props.getProperty("STAGING_HOST");
|
||||
***REMOVED***
|
||||
|
||||
public static void main(String[] args) throws FileNotFoundException ***REMOVED***
|
||||
ServiceConfiguration a = new ServiceConfiguration();
|
||||
System.out.println(a.getStagingHost());
|
||||
System.out.println(a.getCSVUrl());
|
||||
***REMOVED***
|
||||
|
||||
|
||||
***REMOVED***
|
||||
|
|
@ -3,3 +3,17 @@
|
|||
STAGING_HOST: dataminer1-devnext.d4science.org
|
||||
SVN_REPO: https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
|
||||
#HAPROXY_CSV: http:***REMOVED***data.d4science.org/Yk4zSFF6V3JOSytNd3JkRDlnRFpDUUR5TnRJZEw2QjRHbWJQNStIS0N6Yz0
|
||||
|
||||
svn.repository = https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube
|
||||
|
||||
svn.rproto.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/proto/algorithms
|
||||
svn.rproto.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_deb_pkgs.txt
|
||||
svn.rproto.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_cran_pkgs.txt
|
||||
svn.rproto.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/test_r_github_pkgs.txt
|
||||
|
||||
|
||||
svn.prod.algorithms-list = /trunk/data-analysis/DataMinerConfiguration/algorithms/prod/algorithms
|
||||
svn.prod.deps-os = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_deb_pkgs.txt
|
||||
svn.prod.deps-cran = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_cran_pkgs.txt
|
||||
svn.prod.deps-github = /trunk/data-analysis/RConfiguration/RPackagesManagement/r_github_pkgs.txt
|
||||
|
||||
|
|
|
|||
|
|
@ -10,4 +10,6 @@
|
|||
|
||||
<!-- <exclude>*</exclude> -->
|
||||
|
||||
<exclude>/api/swagger.*</exclude>
|
||||
|
||||
</application>
|
||||
|
|
@ -11,14 +11,32 @@
|
|||
<servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
|
||||
<init-param>
|
||||
<param-name>jersey.config.server.provider.packages</param-name>
|
||||
<param-value>org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
|
||||
<param-value>
|
||||
io.swagger.jaxrs.listing,
|
||||
org.gcube.dataanalysis.dataminer.poolmanager.rest</param-value>
|
||||
</init-param>
|
||||
<load-on-startup>1</load-on-startup>
|
||||
</servlet>
|
||||
|
||||
<servlet-mapping>
|
||||
<servlet-name>REST-API</servlet-name>
|
||||
<url-pattern>/rest/*</url-pattern>
|
||||
<url-pattern>/api/*</url-pattern>
|
||||
</servlet-mapping>
|
||||
|
||||
<!--
|
||||
<servlet>
|
||||
<servlet-name>Jersey2Config</servlet-name>
|
||||
<servlet-class>io.swagger.jersey.config.JerseyJaxrsConfig</servlet-class>
|
||||
<init-param>
|
||||
<param-name>api.version</param-name>
|
||||
<param-value>1.0.0</param-value>
|
||||
</init-param>
|
||||
<init-param>
|
||||
<param-name>swagger.api.basepath</param-name>
|
||||
<param-value>http:***REMOVED***localhost:8080/api</param-value>
|
||||
</init-param>
|
||||
<load-on-startup>2</load-on-startup>
|
||||
</servlet>
|
||||
-->
|
||||
|
||||
</web-app>
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import org.gcube.dataanalysis.dataminer.poolmanager.service.StagingJob;
|
|||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.ClusterBuilder;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
|
||||
***REMOVED***
|
||||
|
||||
public class JobTest ***REMOVED***
|
||||
|
|
@ -19,13 +20,13 @@ public class JobTest ***REMOVED***
|
|||
|
||||
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
|
||||
SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
||||
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration());
|
||||
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
|
||||
|
||||
***REMOVED***test phase
|
||||
Cluster stagingCluster = ClusterBuilder.getStagingDataminerCluster();
|
||||
Cluster rProtoCluster = ClusterBuilder.getRProtoCluster();
|
||||
DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster);
|
||||
DMPMJob job = new StagingJob(svnUpdater, algo, stagingCluster, rProtoCluster, ScopeProvider.instance.get());
|
||||
job.start();
|
||||
|
||||
***REMOVED***release phase
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
package org.gcube.dataanalysis.dataminerpoolmanager;
|
||||
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.ServiceConfiguration;
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
|
||||
/**
|
||||
* Created by ggiammat on 5/17/17.
|
||||
*/
|
||||
public class SVNTests ***REMOVED***
|
||||
|
||||
|
||||
public static void main(String[] args) throws SVNException, IOException, InterruptedException ***REMOVED***
|
||||
|
||||
|
||||
SVNUpdater svnUpdater = new SVNUpdater(new ServiceConfiguration("/home/ggiammat/tmp/dmpm.properties"));
|
||||
Algorithm algo = AlgorithmBuilder.create("http:***REMOVED***data.d4science.org/dENQTTMxdjNZcGRpK0NHd2pvU0owMFFzN0VWemw3Zy9HbWJQNStIS0N6Yz0", "ICHTHYOP_MODEL");
|
||||
|
||||
|
||||
svnUpdater.updateRPRotoDeps(algo);
|
||||
***REMOVED***
|
||||
***REMOVED***
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
package org.gcube.dataanalysis.dataminerpoolmanager;
|
||||
|
||||
***REMOVED***
|
||||
import java.util.LinkedList;
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
import org.gcube.dataanalysis.dataminer.poolmanager.util.SVNUpdater;
|
||||
***REMOVED***
|
||||
|
||||
public class SVNUpdaterTest ***REMOVED***
|
||||
|
||||
public static void main(String[] args) throws SVNException, IOException ***REMOVED***
|
||||
ScopeProvider.instance.set("/d4science.research-infrastructures.eu/gCubeApps/RPrototypingLab");
|
||||
|
||||
SVNUpdater svnUpdater = new SVNUpdater("https:***REMOVED***svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/");
|
||||
|
||||
String test = "testDep";
|
||||
List<String> ldep = new LinkedList<>();
|
||||
ldep.add(test);
|
||||
|
||||
svnUpdater.updateSVN("test_r_cran_pkgs.txt", ldep);
|
||||
***REMOVED***
|
||||
|
||||
***REMOVED***
|
||||
Loading…
Reference in New Issue