gCubeSystem
/
dataminer-pool-manager
16
0
Fork 0
Code Issues Pull Requests Releases 2 Wiki Activity

git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@147195 82a268e6-3cf1-43bd-a215-b396298e98cf

This commit is contained in:
Nunzio Andrea Galante 2017-04-26 15:39:24 +00:00
parent 8e5027e67a
commit a07d71b392
23 changed files with 149 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java
View File

@ -174,7 +174,7 @@ public class AnsibleWorker ***REMOVED***
SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url));
ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager();
repository.setAuthenticationManager(authManager);
System.out.println(repository.getLocation());
***REMOVED***System.out.println(repository.getLocation());
return repository;
***REMOVED***

36
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java
View File

@ -3,6 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge;
import java.io.File;
import java.io.FileOutputStream;
***REMOVED***
import java.io.InputStream;
import java.io.PrintStream;
import java.util.Collection;
***REMOVED***
@ -44,7 +45,7 @@ public class AnsibleBridge ***REMOVED***
private String dpmRoot;
public AnsibleBridge() ***REMOVED***
this(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager");
this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager");
***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager"));
***REMOVED***
@ -58,9 +59,9 @@ public class AnsibleBridge ***REMOVED***
***REMOVED*** generate root
new File(dpmRoot).mkdirs();
***REMOVED*** 'template' is for template roles
this.getTemplatesDir().mkdirs();
***REMOVED***this.getTemplatesDir().mkdirs();
***REMOVED*** 'static' is for custom roles
this.getCustomDir().mkdirs();
***REMOVED***this.getCustomDir().mkdirs();
***REMOVED*** 'work' is for temporary working directories
this.getWorkDir().mkdirs();
***REMOVED***
@ -69,13 +70,24 @@ public class AnsibleBridge ***REMOVED***
return new File(this.dpmRoot, "work");
***REMOVED***
private File getTemplatesDir() ***REMOVED***
return new File(this.dpmRoot, "templates");
***REMOVED***
***REMOVED*** private String getTemplatesDir() ***REMOVED***
***REMOVED*** String input = null;
***REMOVED*** input = AnsibleBridge.class.getClassLoader().getResource("templates").getPath();
***REMOVED*** return input;
***REMOVED*** ***REMOVED***
***REMOVED***
***REMOVED***
***REMOVED*** private String getCustomDir() ***REMOVED***
***REMOVED*** String input = null;
***REMOVED*** input = AnsibleBridge.class.getClassLoader().getResource("custom").getPath();
***REMOVED*** return input;
***REMOVED*** ***REMOVED***
private File getCustomDir() ***REMOVED***
return new File(this.dpmRoot, "custom");
***REMOVED***
public AnsibleWorker createWorker() ***REMOVED***
File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString());
@ -221,15 +233,15 @@ public class AnsibleBridge ***REMOVED***
private TemplateManager getTemplateManager() ***REMOVED***
return new TemplateManager(this.dpmRoot+"/templates");
return new TemplateManager();
***REMOVED***
private CustomRoleManager getCustomRoleManager() ***REMOVED***
return new CustomRoleManager(this.dpmRoot+"/custom");
return new CustomRoleManager();
***REMOVED***
private StaticRoleManager getStaticRoleManager() ***REMOVED***
return new StaticRoleManager(this.dpmRoot+"/static");
return new StaticRoleManager();
***REMOVED***
/**

9
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java
View File

@ -5,18 +5,21 @@ import java.io.File;
import java.util.NoSuchElementException;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
public class CustomRoleManager ***REMOVED***
private String root;
public CustomRoleManager(String root) ***REMOVED***
this.root = root;
public CustomRoleManager() ***REMOVED***
***REMOVED***
public String getRoot() ***REMOVED***
return this.root;
String input = AnsibleBridge.class.getClassLoader().getResource("custom").getPath();
return input;
***REMOVED***
public Role getRole(String roleName) throws NoSuchElementException ***REMOVED***

12
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java
View File

@ -2,28 +2,30 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template;
import java.io.File;
***REMOVED***
import java.io.InputStream;
import java.util.Collection;
import java.util.Vector;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
public class StaticRoleManager ***REMOVED***
private String root;
public StaticRoleManager(String root) ***REMOVED***
this.root = root;
public StaticRoleManager() ***REMOVED***
***REMOVED***
public String getRoot() ***REMOVED***
return this.root;
String input = AnsibleBridge.class.getClassLoader().getResource("static").getPath();
return input;
***REMOVED***
public Collection<Role> getStaticRoles() ***REMOVED***
Collection<Role> out = new Vector<>();
for(File f: new File(this.getRoot()).listFiles()) ***REMOVED***
try ***REMOVED***
try ***REMOVED***
out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f));
***REMOVED*** catch(IOException e) ***REMOVED***
e.printStackTrace();

10
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java
View File

@ -7,19 +7,19 @@ import java.util.NoSuchElementException;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role;
import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge;
import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper;
import org.stringtemplate.v4.ST;
public class TemplateManager ***REMOVED***
private String root;
public TemplateManager(String root) ***REMOVED***
this.root = root;
public TemplateManager() ***REMOVED***
***REMOVED***
public String getTemplateRoot() ***REMOVED***
return this.root;
String input = AnsibleBridge.class.getClassLoader().getResource("templates").getPath();
return input;
***REMOVED***
/**

2
src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java
View File

@ -542,7 +542,7 @@ public class DataminerPoolManager implements PoolManager ***REMOVED***
public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException ***REMOVED***
File path = new File(System.getProperty("user.home") + File.separator + "/gcube/dataminer-pool-manager/work/"
File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/"
+ a + File.separator + "logs");
path.mkdirs();
File n = new File(path + File.separator + a);

4
src/main/resources/custom/custom-flr-Flash/meta/main.yaml Normal file
View File

@ -0,0 +1,4 @@
#---
#dependencies:
# <dependencies>
# - ***REMOVED*** role: digest ***REMOVED***

4
src/main/resources/custom/custom-flr-Flash/tasks/main.yaml Normal file
View File

@ -0,0 +1,4 @@
# tasks file for r
---
- name: Install custom algorithm <name>
command: Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('<name>');"

4
src/main/resources/static/gcube-algorithm-installer/meta/main.yaml Normal file
View File

@ -0,0 +1,4 @@
dependencies:
# - ***REMOVED*** role: gcube-dataminer ***REMOVED***
# - ***REMOVED*** role: os-unzip ***REMOVED***
# - ***REMOVED*** role: os-java-1.7.0 ***REMOVED***

31
src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml Normal file
View File

@ -0,0 +1,31 @@
---
- name: remove previous installer (if any)
file:
path: /home/gcube/algorithmInstaller
state: absent
#- name: remove previous installer.zip (if any)
# file:
# path: /home/dpm/algorithmInstaller.zip
# state: absent
#- name: download the installer zip
# get_url:
# url: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip
# dest: /home/dpm/algorithmInstaller.zip
# validate_certs: no
- name: download and unzip the package
unarchive:
src: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip
dest: /home/gcube
remote_src: yes
validate_certs: no
- name: change flags
file:
path: /home/gcube/algorithmInstaller
mode: 0777
state: directory
recurse: yes

2
src/main/resources/static/gcube-dataminer/meta/main.yaml Normal file
View File

@ -0,0 +1,2 @@
dependencies:
- ***REMOVED*** role: gcube-ghn ***REMOVED***

4
src/main/resources/static/gcube-dataminer/tasks/main.yaml Normal file
View File

@ -0,0 +1,4 @@
---
- name: Install the 'DATAMINER' package
shell: echo 'installing DATAMINER'

7
src/main/resources/static/gcube-ghn/tasks/main.yaml Normal file
View File

@ -0,0 +1,7 @@
---
- name: Install the 'GHN' package
shell: echo 'installing GHN'
# TODO: handler to start container
# TODO: handler to stop container

7
src/main/resources/static/os-java-1.7.0/tasks/main.yaml Normal file
View File

@ -0,0 +1,7 @@
---
- name: adding OS Package java-1.7.0
package:
name: java-1.7.0-openjdk
state: "***REMOVED******REMOVED***os_package_state***REMOVED******REMOVED***"
become: true

1
src/main/resources/static/os-r/tasks/main.yaml Normal file
View File

@ -0,0 +1 @@
#to fill with R installation steps

7
src/main/resources/static/os-unzip/tasks/main.yaml Normal file
View File

@ -0,0 +1,7 @@
---
- name: adding OS Package unzip
package:
name: unzip
state: "***REMOVED******REMOVED***os_package_state***REMOVED******REMOVED***"
become: true

4
src/main/resources/templates/cran-package-add/meta/main.yaml Normal file
View File

@ -0,0 +1,4 @@
---
#dependencies:
# <dependencies>
# - ***REMOVED*** role: os-r ***REMOVED***

4
src/main/resources/templates/cran-package-add/tasks/main.yaml Normal file
View File

@ -0,0 +1,4 @@
# tasks file for r
---
- name: Install algorithm <name>
command: Rscript -e "install.packages('<name>', repos=c('http:***REMOVED***ftp.heanet.ie/mirrors/cran.r-project.org/'))"

4
src/main/resources/templates/gcube-algorithm-add/meta/main.yaml Normal file
View File

@ -0,0 +1,4 @@
---
dependencies:
<dependencies>
- ***REMOVED*** role: gcube-algorithm-installer ***REMOVED***

6
src/main/resources/templates/gcube-algorithm-add/tasks/main.yaml Normal file
View File

@ -0,0 +1,6 @@
---
- name: Install algorithm <name>
shell: /home/gcube/algorithmInstaller/addAlgorithm.sh <name> <category> <class> <vre> <atype> <skipjava> <packageurl> "<description>"
args:
chdir: /home/gcube/algorithmInstaller

4
src/main/resources/templates/github-package-add/meta/main.yaml Normal file
View File

@ -0,0 +1,4 @@
---
#dependencies:
# <dependencies>
# - ***REMOVED*** role: os-r ***REMOVED***

4
src/main/resources/templates/github-package-add/tasks/main.yaml Normal file
View File

@ -0,0 +1,4 @@
# tasks file for r
---
- name: Install algorithm <name>
command: Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('<name>');"

8
src/main/resources/templates/os-package-add/tasks/main.yaml Normal file
View File

@ -0,0 +1,8 @@
---
- name: adding OS Package <name>
package:
name: <name>
state: "***REMOVED******REMOVED***os_package_state***REMOVED******REMOVED***"
become: true