gCubeSystem
/
dataminer-pool-manager
15
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@169817 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Ciro Formisano 2018-07-18 09:21:17 +00:00
parent 1d58cc013a
commit 43bc5d5748
9 changed files with 158 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<packaging>war</packaging>
<version>2.4.0-SNAPSHOT</version>
<version>2.5.0-SNAPSHOT</version>
<name>dataminer-pool-manager</name>
<description>

85
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java
View File

@ -5,33 +5,35 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ConfigurationImpl.CONFIGURATIONS;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ConfigurationImpl.REPOSITORIES;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations.AbstractConfiguration;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class ClientConfigurationCache
{
private Map<String, String> repositories;
private Logger logger;
private SVNRepository svnRepository;
private Map<String, AbstractConfiguration> configurations;
private Map<String, Long> repoTimeouts,
configurationsTimeouts;
private long svnRepositoryTimeout;
private Map<String, Long> configurationsTimeouts;
private final long duration = 120000; //2 minutes
public ClientConfigurationCache ()
ClientConfigurationCache ()
{
this.repositories = new HashMap<>();
this.logger = LoggerFactory.getLogger(ClientConfigurationCache.class);
this.svnRepository = null;
this.configurations = new HashMap<>();
this.repoTimeouts = new HashMap<>();
this.svnRepositoryTimeout = 0;
this.configurationsTimeouts = new HashMap<>();
}
@ -50,40 +52,69 @@ public class ClientConfigurationCache
}
public String getRepository (REPOSITORIES repository)
public SVNRepository getSVNRepository ()
{
Long time = this.repoTimeouts.get(repository);
long currentTime = new Date().getTime();
if (time == null || currentTime > time+this.duration)
if (this.svnRepositoryTimeout == 0 || currentTime > this.svnRepositoryTimeout+this.duration)
{
this.repositories.put(repository.toString(), getRepository (repository.toString()));
this.repoTimeouts.put(repository.toString(), currentTime);
this.svnRepository = queryForRepository();
this.svnRepositoryTimeout = currentTime;
}
return this.repositories.get(repository.toString());
return this.svnRepository;
}
private String getRepository(String type) {
String ghost = "";
private SVNRepository queryForRepository()
{
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq 'DMPMConfigurator'").setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
for (GenericResource a : ds) {
ghost = a.profile().body().getElementsByTagName(type).item(0).getTextContent();
Iterator<GenericResource> resourcesIterator = ds.iterator();
SVNRepository response = null;
while (resourcesIterator.hasNext() && response == null)
{
GenericResource resource = resourcesIterator.next();
String repositoryURL = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_URL).item(0).getTextContent();
if (repositoryURL != null)
{
String repositoryPath = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_PATH).item(0).getTextContent();
String repositoryUsername = null;
String repositoryPassword = null;
try
{
repositoryUsername = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_USERNAME).item(0).getTextContent();
repositoryPassword = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_PASSWORD).item(0).getTextContent();
if (repositoryUsername != null && repositoryUsername.trim() == "") repositoryUsername = null;
if (repositoryPassword != null && repositoryPassword.trim() == "") repositoryPassword = null;
this.logger.debug("Repository username "+repositoryUsername);
this.logger.debug("Repository password "+repositoryPassword);
} catch (Exception e)
{
this.logger.debug("SVN Username and password not present");
}
this.logger.debug("SVN Repository URL: "+repositoryURL);
this.logger.debug("SVN Repository path: "+repositoryPath);
response = new SVNRepository(repositoryURL, repositoryPath,repositoryUsername, repositoryPassword);
}
}
return ghost.trim ();
// TODO Auto-generated method stub
return response;
}
private AbstractConfiguration getConfiguration (AbstractConfiguration type)

4
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java
View File

@ -27,9 +27,7 @@ public interface Configuration {
public String getSVNWCDepsList();
public String getSVNRepository();
public String getSVNMainAlgoRepo();
public SVNRepository getSVNRepository();
public String getGhostAlgoDirectory();

39
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java
View File

@ -30,21 +30,21 @@ public class ConfigurationImpl implements Configuration {
}
enum REPOSITORIES {
REPO ("svn.repository"),
MAIN_ALGO ("svn.algo.main.repo");
private String type;
private REPOSITORIES(String type) {
this.type = type;
}
@Override
public String toString() {
return this.type;
}
}
// enum REPOSITORIES {
// REPO ("svn.repository"),
// MAIN_ALGO ("svn.algo.main.repo");
//
// private String type;
//
// private REPOSITORIES(String type) {
// this.type = type;
// }
//
// @Override
// public String toString() {
// return this.type;
// }
// }
private CONFIGURATIONS type;
@ -129,17 +129,12 @@ public class ConfigurationImpl implements Configuration {
}
@Override
public String getSVNRepository()
public SVNRepository getSVNRepository()
{
return this.cache.getRepository(REPOSITORIES.REPO);
return this.cache.getSVNRepository();
}
@Override
public String getSVNMainAlgoRepo() {
return this.cache.getRepository(REPOSITORIES.MAIN_ALGO);
}
@Override
public String getGhostAlgoDirectory() {

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java
View File

@ -106,7 +106,7 @@ public class DMPMClientConfiguratorManager
System.out.println("RESULT 1"+a.getStagingConfiguration().getSVNCRANDepsList());
System.out.println("RESULT 2"+a.getProductionConfiguration().getRepository());
System.out.println("RESULT 3"+a.getStagingConfiguration().getSVNMainAlgoRepo());
System.out.println("RESULT 3"+a.getStagingConfiguration().getSVNRepository().getPath());
//System.out.println(a.getRepo());
//System.out.println(a.getAlgoRepo());
//System.out.println(a.getSVNRepo());

53
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java Executable file
View File

@ -0,0 +1,53 @@
package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration;
public class SVNRepository
{
static final String REPOSITORY_URL = "svn.repository",
REPOSITORY_PATH = "svn.algo.main.repo",
REPOSITORY_USERNAME = "svn.repository.username",
REPOSITORY_PASSWORD = "svn.repository.password";
private String baseUrl,
path,
username,
password;
SVNRepository(String baseUrl, String path, String username, String password) {
this.baseUrl = baseUrl;
this.path = path;
this.username = username;
this.password = password;
}
SVNRepository(String baseUrl, String path) {
this (baseUrl, path, null, null);
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public String getBaseUrl() {
return baseUrl;
}
public String getPath() {
return path;
}
}

4
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java
View File

@ -21,9 +21,6 @@ public abstract class AbstractConfiguration
private String depsWindowsCompiled;
@XmlElement (name="host")
public String getHost() {
return host;
@ -48,6 +45,7 @@ public abstract class AbstractConfiguration
this.softwareRepo = softwareRepo;
}
@XmlElement (name="ghost-repo")
public String getGhostRepo() {
return ghostRepo;

22
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java
View File

@ -1,6 +1,8 @@
package org.gcube.dataanalysis.dataminer.poolmanager.util;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.tmatesoft.svn.core.SVNException;
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
@ -12,11 +14,27 @@ public class SVNRepositoryManager {
private SVNRepository svnRepository;
private static SVNRepositoryManager instance;
private Logger logger;
private SVNRepositoryManager (Configuration configuration) throws SVNException
{
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
this.logger = LoggerFactory.getLogger(SVNRepositoryManager.class);
org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.SVNRepository repository = configuration.getSVNRepository();
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(repository.getBaseUrl()));
ISVNAuthenticationManager authManager = null;
if (repository.getUsername() == null)
{
this.logger.debug("Using SVN default credentials");
authManager = SVNWCUtil.createDefaultAuthenticationManager();
}
else
{
this.logger.debug("Using IS credentials");
authManager = SVNWCUtil.createDefaultAuthenticationManager(repository.getUsername(),repository.getPassword());
}
this.svnRepository.setAuthenticationManager(authManager);
}

14
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java
View File

@ -13,17 +13,6 @@ public class CheckMethodStaging extends CheckMethod{
super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration());
}
@Override
protected void copyFromDmToSVN(File a) throws Exception{
super.copyFromDmToSVN(a, new SVNUpdaterStaging());
@ -31,9 +20,6 @@ public class CheckMethodStaging extends CheckMethod{
}
public static void main(String[] args) throws Exception {
// ServiceConfiguration a = new ServiceConfiguration();
// System.out.println(a.getStagingHost());