git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@147195 82a268e6-3cf1-43bd-a215-b396298e98cf
This commit is contained in:
parent
830080e1ea
commit
34b1d254a4
|
@ -174,7 +174,7 @@ public class AnsibleWorker {
|
||||||
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
|
||||||
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
|
||||||
repository.setAuthenticationManager(authManager);
|
repository.setAuthenticationManager(authManager);
|
||||||
System.out.println(repository.getLocation());
|
***REMOVED***System.out.println(repository.getLocation());
|
||||||
|
|
||||||
return repository;
|
return repository;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
|
@ -3,6 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileOutputStream;
|
import java.io.FileOutputStream;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
import java.io.InputStream;
|
||||||
import java.io.PrintStream;
|
import java.io.PrintStream;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
@ -44,7 +45,7 @@ public class AnsibleBridge {
|
||||||
private String dpmRoot;
|
private String dpmRoot;
|
||||||
|
|
||||||
public AnsibleBridge() {
|
public AnsibleBridge() {
|
||||||
this(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
|
this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager");
|
||||||
***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager"));
|
***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager"));
|
||||||
|
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
@ -58,9 +59,9 @@ public class AnsibleBridge {
|
||||||
***REMOVED*** generate root
|
***REMOVED*** generate root
|
||||||
new File(dpmRoot).mkdirs();
|
new File(dpmRoot).mkdirs();
|
||||||
***REMOVED*** 'template' is for template roles
|
***REMOVED*** 'template' is for template roles
|
||||||
this.getTemplatesDir().mkdirs();
|
***REMOVED***this.getTemplatesDir().mkdirs();
|
||||||
***REMOVED*** 'static' is for custom roles
|
***REMOVED*** 'static' is for custom roles
|
||||||
this.getCustomDir().mkdirs();
|
***REMOVED***this.getCustomDir().mkdirs();
|
||||||
***REMOVED*** 'work' is for temporary working directories
|
***REMOVED*** 'work' is for temporary working directories
|
||||||
this.getWorkDir().mkdirs();
|
this.getWorkDir().mkdirs();
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
@ -69,13 +70,24 @@ public class AnsibleBridge {
|
||||||
return new File(this.dpmRoot, "work");
|
return new File(this.dpmRoot, "work");
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
private File getTemplatesDir() {
|
***REMOVED*** private String getTemplatesDir() {
|
||||||
return new File(this.dpmRoot, "templates");
|
***REMOVED*** String input = null;
|
||||||
***REMOVED***
|
***REMOVED*** input = AnsibleBridge.class.getClassLoader().getResource("templates").getPath();
|
||||||
|
***REMOVED*** return input;
|
||||||
|
***REMOVED*** ***REMOVED***
|
||||||
|
***REMOVED***
|
||||||
|
***REMOVED***
|
||||||
|
***REMOVED*** private String getCustomDir() {
|
||||||
|
***REMOVED*** String input = null;
|
||||||
|
***REMOVED*** input = AnsibleBridge.class.getClassLoader().getResource("custom").getPath();
|
||||||
|
***REMOVED*** return input;
|
||||||
|
***REMOVED*** ***REMOVED***
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
private File getCustomDir() {
|
|
||||||
return new File(this.dpmRoot, "custom");
|
|
||||||
***REMOVED***
|
|
||||||
|
|
||||||
public AnsibleWorker createWorker() {
|
public AnsibleWorker createWorker() {
|
||||||
File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString());
|
File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString());
|
||||||
|
@ -221,15 +233,15 @@ public class AnsibleBridge {
|
||||||
|
|
||||||
|
|
||||||
private TemplateManager getTemplateManager() {
|
private TemplateManager getTemplateManager() {
|
||||||
return new TemplateManager(this.dpmRoot+"/templates");
|
return new TemplateManager();
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
private CustomRoleManager getCustomRoleManager() {
|
private CustomRoleManager getCustomRoleManager() {
|
||||||
return new CustomRoleManager(this.dpmRoot+"/custom");
|
return new CustomRoleManager();
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
private StaticRoleManager getStaticRoleManager() {
|
private StaticRoleManager getStaticRoleManager() {
|
||||||
return new StaticRoleManager(this.dpmRoot+"/static");
|
return new StaticRoleManager();
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -5,18 +5,21 @@ import java.io.File;
|
||||||
import java.util.NoSuchElementException;
|
import java.util.NoSuchElementException;
|
||||||
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
|
||||||
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
|
||||||
|
|
||||||
public class CustomRoleManager {
|
public class CustomRoleManager {
|
||||||
|
|
||||||
private String root;
|
private String root;
|
||||||
|
|
||||||
public CustomRoleManager(String root) {
|
public CustomRoleManager() {
|
||||||
this.root = root;
|
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
public String getRoot() {
|
public String getRoot() {
|
||||||
return this.root;
|
String input = AnsibleBridge.class.getClassLoader().getResource("custom").getPath();
|
||||||
|
|
||||||
|
return input;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
public Role getRole(String roleName) throws NoSuchElementException {
|
public Role getRole(String roleName) throws NoSuchElementException {
|
||||||
|
|
|
@ -2,22 +2,24 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
|
||||||
|
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
import java.io.InputStream;
|
||||||
import java.util.Collection;
|
import java.util.Collection;
|
||||||
import java.util.Vector;
|
import java.util.Vector;
|
||||||
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
|
||||||
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
|
||||||
|
|
||||||
public class StaticRoleManager {
|
public class StaticRoleManager {
|
||||||
|
|
||||||
private String root;
|
|
||||||
|
|
||||||
public StaticRoleManager(String root) {
|
public StaticRoleManager() {
|
||||||
this.root = root;
|
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
public String getRoot() {
|
public String getRoot() {
|
||||||
return this.root;
|
String input = AnsibleBridge.class.getClassLoader().getResource("static").getPath();
|
||||||
|
return input;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
public Collection<Role> getStaticRoles() {
|
public Collection<Role> getStaticRoles() {
|
||||||
|
|
|
@ -7,19 +7,19 @@ import java.util.NoSuchElementException;
|
||||||
|
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
|
||||||
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
|
||||||
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
|
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
|
||||||
import org.stringtemplate.v4.ST;
|
import org.stringtemplate.v4.ST;
|
||||||
|
|
||||||
public class TemplateManager {
|
public class TemplateManager {
|
||||||
|
|
||||||
private String root;
|
public TemplateManager() {
|
||||||
|
|
||||||
public TemplateManager(String root) {
|
|
||||||
this.root = root;
|
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
public String getTemplateRoot() {
|
public String getTemplateRoot() {
|
||||||
return this.root;
|
String input = AnsibleBridge.class.getClassLoader().getResource("templates").getPath();
|
||||||
|
return input;
|
||||||
***REMOVED***
|
***REMOVED***
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
@ -542,7 +542,7 @@ public class DataminerPoolManager implements PoolManager {
|
||||||
|
|
||||||
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
|
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException {
|
||||||
|
|
||||||
File path = new File(System.getProperty("user.home") + File.separator + "/gcube/dataminer-pool-manager/work/"
|
File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/"
|
||||||
+ a + File.separator + "logs");
|
+ a + File.separator + "logs");
|
||||||
path.mkdirs();
|
path.mkdirs();
|
||||||
File n = new File(path + File.separator + a);
|
File n = new File(path + File.separator + a);
|
||||||
|
|
|
@ -0,0 +1,4 @@
|
||||||
|
#---
|
||||||
|
#dependencies:
|
||||||
|
# <dependencies>
|
||||||
|
# - { role: digest ***REMOVED***
|
|
@ -0,0 +1,4 @@
|
||||||
|
# tasks file for r
|
||||||
|
---
|
||||||
|
- name: Install custom algorithm <name>
|
||||||
|
command: Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('<name>');"
|
|
@ -0,0 +1,4 @@
|
||||||
|
dependencies:
|
||||||
|
# - { role: gcube-dataminer ***REMOVED***
|
||||||
|
# - { role: os-unzip ***REMOVED***
|
||||||
|
# - { role: os-java-1.7.0 ***REMOVED***
|
|
@ -0,0 +1,31 @@
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: remove previous installer (if any)
|
||||||
|
file:
|
||||||
|
path: /home/gcube/algorithmInstaller
|
||||||
|
state: absent
|
||||||
|
|
||||||
|
#- name: remove previous installer.zip (if any)
|
||||||
|
# file:
|
||||||
|
# path: /home/dpm/algorithmInstaller.zip
|
||||||
|
# state: absent
|
||||||
|
|
||||||
|
#- name: download the installer zip
|
||||||
|
# get_url:
|
||||||
|
# url: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip
|
||||||
|
# dest: /home/dpm/algorithmInstaller.zip
|
||||||
|
# validate_certs: no
|
||||||
|
|
||||||
|
- name: download and unzip the package
|
||||||
|
unarchive:
|
||||||
|
src: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip
|
||||||
|
dest: /home/gcube
|
||||||
|
remote_src: yes
|
||||||
|
validate_certs: no
|
||||||
|
|
||||||
|
- name: change flags
|
||||||
|
file:
|
||||||
|
path: /home/gcube/algorithmInstaller
|
||||||
|
mode: 0777
|
||||||
|
state: directory
|
||||||
|
recurse: yes
|
|
@ -0,0 +1,2 @@
|
||||||
|
dependencies:
|
||||||
|
- { role: gcube-ghn ***REMOVED***
|
|
@ -0,0 +1,4 @@
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: Install the 'DATAMINER' package
|
||||||
|
shell: echo 'installing DATAMINER'
|
|
@ -0,0 +1,7 @@
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: Install the 'GHN' package
|
||||||
|
shell: echo 'installing GHN'
|
||||||
|
|
||||||
|
# TODO: handler to start container
|
||||||
|
# TODO: handler to stop container
|
|
@ -0,0 +1,7 @@
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: adding OS Package java-1.7.0
|
||||||
|
package:
|
||||||
|
name: java-1.7.0-openjdk
|
||||||
|
state: "{{os_package_state***REMOVED******REMOVED***"
|
||||||
|
become: true
|
|
@ -0,0 +1 @@
|
||||||
|
#to fill with R installation steps
|
|
@ -0,0 +1,7 @@
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: adding OS Package unzip
|
||||||
|
package:
|
||||||
|
name: unzip
|
||||||
|
state: "{{os_package_state***REMOVED******REMOVED***"
|
||||||
|
become: true
|
|
@ -0,0 +1,4 @@
|
||||||
|
---
|
||||||
|
#dependencies:
|
||||||
|
# <dependencies>
|
||||||
|
# - { role: os-r ***REMOVED***
|
|
@ -0,0 +1,4 @@
|
||||||
|
# tasks file for r
|
||||||
|
---
|
||||||
|
- name: Install algorithm <name>
|
||||||
|
command: Rscript -e "install.packages('<name>', repos=c('http:***REMOVED***ftp.heanet.ie/mirrors/cran.r-project.org/'))"
|
|
@ -0,0 +1,4 @@
|
||||||
|
---
|
||||||
|
dependencies:
|
||||||
|
<dependencies>
|
||||||
|
- { role: gcube-algorithm-installer ***REMOVED***
|
|
@ -0,0 +1,6 @@
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: Install algorithm <name>
|
||||||
|
shell: /home/gcube/algorithmInstaller/addAlgorithm.sh <name> <category> <class> <vre> <atype> <skipjava> <packageurl> "<description>"
|
||||||
|
args:
|
||||||
|
chdir: /home/gcube/algorithmInstaller
|
|
@ -0,0 +1,4 @@
|
||||||
|
---
|
||||||
|
#dependencies:
|
||||||
|
# <dependencies>
|
||||||
|
# - { role: os-r ***REMOVED***
|
|
@ -0,0 +1,4 @@
|
||||||
|
# tasks file for r
|
||||||
|
---
|
||||||
|
- name: Install algorithm <name>
|
||||||
|
command: Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('<name>');"
|
|
@ -0,0 +1,8 @@
|
||||||
|
---
|
||||||
|
|
||||||
|
- name: adding OS Package <name>
|
||||||
|
package:
|
||||||
|
name: <name>
|
||||||
|
state: "{{os_package_state***REMOVED******REMOVED***"
|
||||||
|
become: true
|
||||||
|
|
Loading…
Reference in New Issue