gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@169817 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Ciro Formisano 2018-07-18 09:21:17 +00:00
parent e8e45abb36
commit 91ffb8e3b7
9 changed files with 158 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<packaging>war</packaging>
<version>2.4.0-SNAPSHOT</version>
<version>2.5.0-SNAPSHOT</version>
<name>dataminer-pool-manager</name>
<description>

87
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java
View File

@ -5,33 +5,35 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
***REMOVED***
import java.util.Map;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ConfigurationImpl.CONFIGURATIONS;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ConfigurationImpl.REPOSITORIES;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations.AbstractConfiguration;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
***REMOVED***
***REMOVED***
public class ClientConfigurationCache
{
private Map<String, String> repositories;
***REMOVED***
private SVNRepository svnRepository;
private Map<String, AbstractConfiguration> configurations;
private Map<String, Long> repoTimeouts,
configurationsTimeouts;
private long svnRepositoryTimeout;
private Map<String, Long> configurationsTimeouts;
private final long duration = 120000; ***REMOVED***2 minutes
public ClientConfigurationCache ()
ClientConfigurationCache ()
{
this.repositories = new HashMap<>();
this.logger = LoggerFactory.getLogger(ClientConfigurationCache.class);
this.svnRepository = null;
this.configurations = new HashMap<>();
this.repoTimeouts = new HashMap<>();
this.svnRepositoryTimeout = 0;
this.configurationsTimeouts = new HashMap<>();
***REMOVED***
@ -50,40 +52,69 @@ public class ClientConfigurationCache
***REMOVED***
public String getRepository (REPOSITORIES repository)
public SVNRepository getSVNRepository ()
{
Long time = this.repoTimeouts.get(repository);
long currentTime = new Date().getTime();
if (time == null || currentTime > time+this.duration)
if (this.svnRepositoryTimeout == 0 || currentTime > this.svnRepositoryTimeout+this.duration)
{
this.repositories.put(repository.toString(), getRepository (repository.toString()));
this.repoTimeouts.put(repository.toString(), currentTime);
this.svnRepository = queryForRepository();
this.svnRepositoryTimeout = currentTime;
***REMOVED***
return this.repositories.get(repository.toString());
return this.svnRepository;
***REMOVED***
private String getRepository(String type) {
String ghost = "";
private SVNRepository queryForRepository()
{
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq 'DMPMConfigurator'").setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
for (GenericResource a : ds) {
ghost = a.profile().body().getElementsByTagName(type).item(0).getTextContent();
Iterator<GenericResource> resourcesIterator = ds.iterator();
SVNRepository response = null;
while (resourcesIterator.hasNext() && response == null)
{
GenericResource resource = resourcesIterator.next();
String repositoryURL = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_URL).item(0).getTextContent();
if (repositoryURL != null)
{
String repositoryPath = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_PATH).item(0).getTextContent();
String repositoryUsername = null;
String repositoryPassword = null;
try
{
repositoryUsername = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_USERNAME).item(0).getTextContent();
repositoryPassword = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_PASSWORD).item(0).getTextContent();
if (repositoryUsername != null && repositoryUsername.trim() == "") repositoryUsername = null;
if (repositoryPassword != null && repositoryPassword.trim() == "") repositoryPassword = null;
this.logger.debug("Repository username "+repositoryUsername);
this.logger.debug("Repository password "+repositoryPassword);
***REMOVED*** catch (Exception e)
{
this.logger.debug("SVN Username and password not present");
***REMOVED***
this.logger.debug("SVN Repository URL: "+repositoryURL);
this.logger.debug("SVN Repository path: "+repositoryPath);
response = new SVNRepository(repositoryURL, repositoryPath,repositoryUsername, repositoryPassword);
***REMOVED***
***REMOVED***
return ghost.trim ();
***REMOVED*** TODO Auto-generated method stub
return response;
***REMOVED***
private AbstractConfiguration getConfiguration (AbstractConfiguration type)

4
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java
View File

@ -27,9 +27,7 @@ public interface Configuration {
public String getSVNWCDepsList();
public String getSVNRepository();
public String getSVNMainAlgoRepo();
public SVNRepository getSVNRepository();
public String getGhostAlgoDirectory();

39
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java
View File

@ -30,21 +30,21 @@ public class ConfigurationImpl implements Configuration {
***REMOVED***
enum REPOSITORIES {
REPO ("svn.repository"),
MAIN_ALGO ("svn.algo.main.repo");
private String type;
private REPOSITORIES(String type) {
this.type = type;
***REMOVED***
@Override
public String toString() {
return this.type;
***REMOVED***
***REMOVED***
***REMOVED*** enum REPOSITORIES {
***REMOVED*** REPO ("svn.repository"),
***REMOVED*** MAIN_ALGO ("svn.algo.main.repo");
***REMOVED***
***REMOVED*** private String type;
***REMOVED***
***REMOVED*** private REPOSITORIES(String type) {
***REMOVED*** this.type = type;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED*** @Override
***REMOVED*** public String toString() {
***REMOVED*** return this.type;
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
private CONFIGURATIONS type;
@ -129,17 +129,12 @@ public class ConfigurationImpl implements Configuration {
***REMOVED***
@Override
public String getSVNRepository()
public SVNRepository getSVNRepository()
{
return this.cache.getRepository(REPOSITORIES.REPO);
return this.cache.getSVNRepository();
***REMOVED***
@Override
public String getSVNMainAlgoRepo() {
return this.cache.getRepository(REPOSITORIES.MAIN_ALGO);
***REMOVED***
@Override
public String getGhostAlgoDirectory() {

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java
View File

@ -106,7 +106,7 @@ public class DMPMClientConfiguratorManager
System.out.println("RESULT 1"+a.getStagingConfiguration().getSVNCRANDepsList());
System.out.println("RESULT 2"+a.getProductionConfiguration().getRepository());
System.out.println("RESULT 3"+a.getStagingConfiguration().getSVNMainAlgoRepo());
System.out.println("RESULT 3"+a.getStagingConfiguration().getSVNRepository().getPath());
***REMOVED***System.out.println(a.getRepo());
***REMOVED***System.out.println(a.getAlgoRepo());
***REMOVED***System.out.println(a.getSVNRepo());

53
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java Executable file
View File

@ -0,0 +1,53 @@
package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration;
public class SVNRepository
{
static final String REPOSITORY_URL = "svn.repository",
REPOSITORY_PATH = "svn.algo.main.repo",
REPOSITORY_USERNAME = "svn.repository.username",
REPOSITORY_PASSWORD = "svn.repository.password";
private String baseUrl,
path,
username,
password;
SVNRepository(String baseUrl, String path, String username, String password) {
this.baseUrl = baseUrl;
this.path = path;
this.username = username;
this.password = password;
***REMOVED***
SVNRepository(String baseUrl, String path) {
this (baseUrl, path, null, null);
***REMOVED***
public String getUsername() {
return username;
***REMOVED***
public void setUsername(String username) {
this.username = username;
***REMOVED***
public String getPassword() {
return password;
***REMOVED***
public void setPassword(String password) {
this.password = password;
***REMOVED***
public String getBaseUrl() {
return baseUrl;
***REMOVED***
public String getPath() {
return path;
***REMOVED***
***REMOVED***

12
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java
View File

@ -6,7 +6,7 @@ import javax.xml.bind.annotation.XmlElement;
public abstract class AbstractConfiguration
{
private String host;
private String host;
private String algorithmsList;
private String softwareRepo;
private String ghostRepo;
@ -19,11 +19,8 @@ public abstract class AbstractConfiguration
private String depsOctave;
private String depsPython;
private String depsWindowsCompiled;
@XmlElement (name="host")
public String getHost() {
return host;
@ -47,7 +44,8 @@ public abstract class AbstractConfiguration
public void setSoftwareRepo(String softwareRepo) {
this.softwareRepo = softwareRepo;
***REMOVED***
@XmlElement (name="ghost-repo")
public String getGhostRepo() {
return ghostRepo;

22
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java
View File

@ -1,6 +1,8 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
@ -12,11 +14,27 @@ public class SVNRepositoryManager {
private SVNRepository svnRepository;
private static SVNRepositoryManager instance;
***REMOVED***
private SVNRepositoryManager (Configuration configuration) throws SVNException
{
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
this.logger = LoggerFactory.getLogger(SVNRepositoryManager.class);
org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.SVNRepository repository = configuration.getSVNRepository();
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(repository.getBaseUrl()));
ISVNAuthenticationManager authManager = null;
if (repository.getUsername() == null)
{
this.logger.debug("Using SVN default credentials");
authManager = SVNWCUtil.createDefaultAuthenticationManager();
***REMOVED***
else
{
this.logger.debug("Using IS credentials");
authManager = SVNWCUtil.createDefaultAuthenticationManager(repository.getUsername(),repository.getPassword());
***REMOVED***
this.svnRepository.setAuthenticationManager(authManager);
***REMOVED***

16
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java
View File

@ -12,17 +12,6 @@ public class CheckMethodStaging extends CheckMethod{
{
super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration());
***REMOVED***
@Override
protected void copyFromDmToSVN(File a) throws Exception{
@ -30,10 +19,7 @@ public class CheckMethodStaging extends CheckMethod{
***REMOVED***
public static void main(String[] args) throws Exception {
***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
***REMOVED*** System.out.println(a.getStagingHost());