gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@169817 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Ciro Formisano 2018-07-18 09:21:17 +00:00
parent ad712aa95a
commit dc58b53b34
9 changed files with 158 additions and 79 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pom.xml
View File

@ -12,7 +12,7 @@
<groupId>org.gcube.dataAnalysis</groupId>
<artifactId>dataminer-pool-manager</artifactId>
<packaging>war</packaging>
<version>2.4.0-SNAPSHOT</version>
<version>2.5.0-SNAPSHOT</version>
<name>dataminer-pool-manager</name>
<description>

87
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ClientConfigurationCache.java
View File

@ -5,33 +5,35 @@ import static org.gcube.resources.discovery.icclient.ICFactory.queryFor;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
***REMOVED***
import java.util.Map;
import org.gcube.common.resources.gcore.GenericResource;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ConfigurationImpl.CONFIGURATIONS;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.ConfigurationImpl.REPOSITORIES;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configurations.AbstractConfiguration;
import org.gcube.resources.discovery.client.api.DiscoveryClient;
import org.gcube.resources.discovery.client.queries.api.SimpleQuery;
***REMOVED***
***REMOVED***
public class ClientConfigurationCache
***REMOVED***
private Map<String, String> repositories;
private Logger logger;
private SVNRepository svnRepository;
private Map<String, AbstractConfiguration> configurations;
private Map<String, Long> repoTimeouts,
configurationsTimeouts;
private long svnRepositoryTimeout;
private Map<String, Long> configurationsTimeouts;
private final long duration = 120000; ***REMOVED***2 minutes
public ClientConfigurationCache ()
ClientConfigurationCache ()
***REMOVED***
this.repositories = new HashMap<>();
this.logger = LoggerFactory.getLogger(ClientConfigurationCache.class);
this.svnRepository = null;
this.configurations = new HashMap<>();
this.repoTimeouts = new HashMap<>();
this.svnRepositoryTimeout = 0;
this.configurationsTimeouts = new HashMap<>();
***REMOVED***
@ -50,40 +52,69 @@ public class ClientConfigurationCache
***REMOVED***
public String getRepository (REPOSITORIES repository)
public SVNRepository getSVNRepository ()
***REMOVED***
Long time = this.repoTimeouts.get(repository);
long currentTime = new Date().getTime();
if (time == null || currentTime > time+this.duration)
if (this.svnRepositoryTimeout == 0 || currentTime > this.svnRepositoryTimeout+this.duration)
***REMOVED***
this.repositories.put(repository.toString(), getRepository (repository.toString()));
this.repoTimeouts.put(repository.toString(), currentTime);
this.svnRepository = queryForRepository();
this.svnRepositoryTimeout = currentTime;
***REMOVED***
return this.repositories.get(repository.toString());
return this.svnRepository;
***REMOVED***
private String getRepository(String type) ***REMOVED***
String ghost = "";
private SVNRepository queryForRepository()
***REMOVED***
SimpleQuery query = queryFor(GenericResource.class);
query.addCondition("$resource/Profile/SecondaryType/text() eq 'DMPMConfigurator'").setResult("$resource");
DiscoveryClient<GenericResource> client = clientFor(GenericResource.class);
List<GenericResource> ds = client.submit(query);
for (GenericResource a : ds) ***REMOVED***
ghost = a.profile().body().getElementsByTagName(type).item(0).getTextContent();
Iterator<GenericResource> resourcesIterator = ds.iterator();
SVNRepository response = null;
while (resourcesIterator.hasNext() && response == null)
***REMOVED***
GenericResource resource = resourcesIterator.next();
String repositoryURL = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_URL).item(0).getTextContent();
if (repositoryURL != null)
***REMOVED***
String repositoryPath = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_PATH).item(0).getTextContent();
String repositoryUsername = null;
String repositoryPassword = null;
try
***REMOVED***
repositoryUsername = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_USERNAME).item(0).getTextContent();
repositoryPassword = resource.profile().body().getElementsByTagName(SVNRepository.REPOSITORY_PASSWORD).item(0).getTextContent();
if (repositoryUsername != null && repositoryUsername.trim() == "") repositoryUsername = null;
if (repositoryPassword != null && repositoryPassword.trim() == "") repositoryPassword = null;
this.logger.debug("Repository username "+repositoryUsername);
this.logger.debug("Repository password "+repositoryPassword);
***REMOVED*** catch (Exception e)
***REMOVED***
this.logger.debug("SVN Username and password not present");
***REMOVED***
this.logger.debug("SVN Repository URL: "+repositoryURL);
this.logger.debug("SVN Repository path: "+repositoryPath);
response = new SVNRepository(repositoryURL, repositoryPath,repositoryUsername, repositoryPassword);
***REMOVED***
***REMOVED***
return ghost.trim ();
***REMOVED***
return response;
***REMOVED***
private AbstractConfiguration getConfiguration (AbstractConfiguration type)

4
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/Configuration.java
View File

@ -27,9 +27,7 @@ public interface Configuration ***REMOVED***
public String getSVNWCDepsList();
public String getSVNRepository();
public String getSVNMainAlgoRepo();
public SVNRepository getSVNRepository();
public String getGhostAlgoDirectory();

39
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/ConfigurationImpl.java
View File

@ -30,21 +30,21 @@ public class ConfigurationImpl implements Configuration ***REMOVED***
***REMOVED***
enum REPOSITORIES ***REMOVED***
REPO ("svn.repository"),
MAIN_ALGO ("svn.algo.main.repo");
private String type;
private REPOSITORIES(String type) ***REMOVED***
this.type = type;
***REMOVED***
***REMOVED***
public String toString() ***REMOVED***
return this.type;
***REMOVED***
***REMOVED***
***REMOVED*** enum REPOSITORIES ***REMOVED***
***REMOVED*** REPO ("svn.repository"),
***REMOVED*** MAIN_ALGO ("svn.algo.main.repo");
***REMOVED***
***REMOVED*** private String type;
***REMOVED***
***REMOVED*** private REPOSITORIES(String type) ***REMOVED***
***REMOVED*** this.type = type;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED******REMOVED***
***REMOVED*** public String toString() ***REMOVED***
***REMOVED*** return this.type;
***REMOVED*** ***REMOVED***
***REMOVED******REMOVED***
private CONFIGURATIONS type;
@ -129,17 +129,12 @@ public class ConfigurationImpl implements Configuration ***REMOVED***
***REMOVED***
***REMOVED***
public String getSVNRepository()
public SVNRepository getSVNRepository()
***REMOVED***
return this.cache.getRepository(REPOSITORIES.REPO);
return this.cache.getSVNRepository();
***REMOVED***
***REMOVED***
public String getSVNMainAlgoRepo() ***REMOVED***
return this.cache.getRepository(REPOSITORIES.MAIN_ALGO);
***REMOVED***
***REMOVED***
public String getGhostAlgoDirectory() ***REMOVED***

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/DMPMClientConfiguratorManager.java
View File

@ -106,7 +106,7 @@ public class DMPMClientConfiguratorManager
System.out.println("RESULT 1"+a.getStagingConfiguration().getSVNCRANDepsList());
System.out.println("RESULT 2"+a.getProductionConfiguration().getRepository());
System.out.println("RESULT 3"+a.getStagingConfiguration().getSVNMainAlgoRepo());
System.out.println("RESULT 3"+a.getStagingConfiguration().getSVNRepository().getPath());
***REMOVED***System.out.println(a.getRepo());
***REMOVED***System.out.println(a.getAlgoRepo());
***REMOVED***System.out.println(a.getSVNRepo());

53
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configuration/SVNRepository.java Executable file
View File

@ -0,0 +1,53 @@
package org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration;
public class SVNRepository
***REMOVED***
static final String REPOSITORY_URL = "svn.repository",
REPOSITORY_PATH = "svn.algo.main.repo",
REPOSITORY_USERNAME = "svn.repository.username",
REPOSITORY_PASSWORD = "svn.repository.password";
private String baseUrl,
path,
username,
password;
SVNRepository(String baseUrl, String path, String username, String password) ***REMOVED***
this.baseUrl = baseUrl;
this.path = path;
this.username = username;
this.password = password;
***REMOVED***
SVNRepository(String baseUrl, String path) ***REMOVED***
this (baseUrl, path, null, null);
***REMOVED***
public String getUsername() ***REMOVED***
return username;
***REMOVED***
public void setUsername(String username) ***REMOVED***
this.username = username;
***REMOVED***
public String getPassword() ***REMOVED***
return password;
***REMOVED***
public void setPassword(String password) ***REMOVED***
this.password = password;
***REMOVED***
public String getBaseUrl() ***REMOVED***
return baseUrl;
***REMOVED***
public String getPath() ***REMOVED***
return path;
***REMOVED***
***REMOVED***

12
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/clients/configurations/AbstractConfiguration.java
View File

@ -6,7 +6,7 @@ import javax.xml.bind.annotation.XmlElement;
public abstract class AbstractConfiguration
***REMOVED***
private String host;
private String host;
private String algorithmsList;
private String softwareRepo;
private String ghostRepo;
@ -19,11 +19,8 @@ public abstract class AbstractConfiguration
private String depsOctave;
private String depsPython;
private String depsWindowsCompiled;
@XmlElement (name="host")
public String getHost() ***REMOVED***
return host;
@ -47,7 +44,8 @@ public abstract class AbstractConfiguration
public void setSoftwareRepo(String softwareRepo) ***REMOVED***
this.softwareRepo = softwareRepo;
***REMOVED***
@XmlElement (name="ghost-repo")
public String getGhostRepo() ***REMOVED***
return ghostRepo;

22
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/SVNRepositoryManager.java
View File

@ -2,6 +2,8 @@ package org.gcube.dataanalysis.dataminer.poolmanager.util;
import org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.Configuration;
***REMOVED***
***REMOVED***
***REMOVED***
import org.tmatesoft.svn.core.SVNURL;
import org.tmatesoft.svn.core.auth.ISVNAuthenticationManager;
import org.tmatesoft.svn.core.io.SVNRepository;
@ -12,11 +14,27 @@ public class SVNRepositoryManager ***REMOVED***
private SVNRepository svnRepository;
private static SVNRepositoryManager instance;
private Logger logger;
private SVNRepositoryManager (Configuration configuration) throws SVNException
***REMOVED***
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(configuration.getSVNRepository()));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
this.logger = LoggerFactory.getLogger(SVNRepositoryManager.class);
org.gcube.dataanalysis.dataminer.poolmanager.clients.configuration.SVNRepository repository = configuration.getSVNRepository();
this.svnRepository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(repository.getBaseUrl()));
ISVNAuthenticationManager authManager = null;
if (repository.getUsername() == null)
***REMOVED***
this.logger.debug("Using SVN default credentials");
authManager = SVNWCUtil.createDefaultAuthenticationManager();
***REMOVED***
else
***REMOVED***
this.logger.debug("Using IS credentials");
authManager = SVNWCUtil.createDefaultAuthenticationManager(repository.getUsername(),repository.getPassword());
***REMOVED***
this.svnRepository.setAuthenticationManager(authManager);
***REMOVED***

16
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/util/impl/CheckMethodStaging.java
View File

@ -12,17 +12,6 @@ public class CheckMethodStaging extends CheckMethod***REMOVED***
***REMOVED***
super (DMPMClientConfiguratorManager.getInstance().getStagingConfiguration());
***REMOVED***
***REMOVED***
protected void copyFromDmToSVN(File a) throws Exception***REMOVED***
@ -30,10 +19,7 @@ public class CheckMethodStaging extends CheckMethod***REMOVED***
***REMOVED***
public static void main(String[] args) throws Exception ***REMOVED***
***REMOVED*** ServiceConfiguration a = new ServiceConfiguration();
***REMOVED*** System.out.println(a.getStagingHost());