From 34b1d254a41da62f0822873d3163d5fe83fea9fe Mon Sep 17 00:00:00 2001 From: Nunzio Andrea Galante Date: Wed, 26 Apr 2017 15:39:24 +0000 Subject: [PATCH] git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/data-analysis/dataminer-pool-manager@147195 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../poolmanager/ansible/AnsibleWorker.java | 2 +- .../ansiblebridge/AnsibleBridge.java | 36 ++++++++++++------- .../template/CustomRoleManager.java | 9 +++-- .../template/StaticRoleManager.java | 12 ++++--- .../template/TemplateManager.java | 10 +++--- .../service/DataminerPoolManager.java | 2 +- .../custom/custom-flr-Flash/meta/main.yaml | 4 +++ .../custom/custom-flr-Flash/tasks/main.yaml | 4 +++ .../gcube-algorithm-installer/meta/main.yaml | 4 +++ .../gcube-algorithm-installer/tasks/main.yaml | 31 ++++++++++++++++ .../static/gcube-dataminer/meta/main.yaml | 2 ++ .../static/gcube-dataminer/tasks/main.yaml | 4 +++ .../static/gcube-ghn/tasks/main.yaml | 7 ++++ .../static/os-java-1.7.0/tasks/main.yaml | 7 ++++ .../resources/static/os-r/tasks/main.yaml | 1 + .../resources/static/os-unzip/tasks/main.yaml | 7 ++++ .../templates/cran-package-add/meta/main.yaml | 4 +++ .../cran-package-add/tasks/main.yaml | 4 +++ .../gcube-algorithm-add/meta/main.yaml | 4 +++ .../gcube-algorithm-add/tasks/main.yaml | 6 ++++ .../github-package-add/meta/main.yaml | 4 +++ .../github-package-add/tasks/main.yaml | 4 +++ .../templates/os-package-add/tasks/main.yaml | 8 +++++ 23 files changed, 149 insertions(+), 27 deletions(-) create mode 100644 src/main/resources/custom/custom-flr-Flash/meta/main.yaml create mode 100644 src/main/resources/custom/custom-flr-Flash/tasks/main.yaml create mode 100644 src/main/resources/static/gcube-algorithm-installer/meta/main.yaml create mode 100644 src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml create mode 100644 src/main/resources/static/gcube-dataminer/meta/main.yaml create mode 100644 src/main/resources/static/gcube-dataminer/tasks/main.yaml create mode 100644 src/main/resources/static/gcube-ghn/tasks/main.yaml create mode 100644 src/main/resources/static/os-java-1.7.0/tasks/main.yaml create mode 100644 src/main/resources/static/os-r/tasks/main.yaml create mode 100644 src/main/resources/static/os-unzip/tasks/main.yaml create mode 100644 src/main/resources/templates/cran-package-add/meta/main.yaml create mode 100644 src/main/resources/templates/cran-package-add/tasks/main.yaml create mode 100644 src/main/resources/templates/gcube-algorithm-add/meta/main.yaml create mode 100644 src/main/resources/templates/gcube-algorithm-add/tasks/main.yaml create mode 100644 src/main/resources/templates/github-package-add/meta/main.yaml create mode 100644 src/main/resources/templates/github-package-add/tasks/main.yaml create mode 100644 src/main/resources/templates/os-package-add/tasks/main.yaml diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java index beaba8d..c9b8b85 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansible/AnsibleWorker.java @@ -174,7 +174,7 @@ public class AnsibleWorker { SVNRepository repository = SVNRepositoryFactory.create(SVNURL.parseURIEncoded(url)); ISVNAuthenticationManager authManager = SVNWCUtil.createDefaultAuthenticationManager(); repository.setAuthenticationManager(authManager); - System.out.println(repository.getLocation()); + ***REMOVED***System.out.println(repository.getLocation()); return repository; ***REMOVED*** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java index c755985..5319d37 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/AnsibleBridge.java @@ -3,6 +3,7 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge; import java.io.File; import java.io.FileOutputStream; ***REMOVED*** +import java.io.InputStream; import java.io.PrintStream; import java.util.Collection; ***REMOVED*** @@ -44,7 +45,7 @@ public class AnsibleBridge { private String dpmRoot; public AnsibleBridge() { - this(System.getProperty("user.home")+File.separator+"/gcube/dataminer-pool-manager"); + this(System.getProperty("user.home")+File.separator+"dataminer-pool-manager"); ***REMOVED***this(System.getProperty("/home/gcube/dataminer-pool-manager")); ***REMOVED*** @@ -58,9 +59,9 @@ public class AnsibleBridge { ***REMOVED*** generate root new File(dpmRoot).mkdirs(); ***REMOVED*** 'template' is for template roles - this.getTemplatesDir().mkdirs(); + ***REMOVED***this.getTemplatesDir().mkdirs(); ***REMOVED*** 'static' is for custom roles - this.getCustomDir().mkdirs(); + ***REMOVED***this.getCustomDir().mkdirs(); ***REMOVED*** 'work' is for temporary working directories this.getWorkDir().mkdirs(); ***REMOVED*** @@ -69,13 +70,24 @@ public class AnsibleBridge { return new File(this.dpmRoot, "work"); ***REMOVED*** - private File getTemplatesDir() { - return new File(this.dpmRoot, "templates"); - ***REMOVED*** +***REMOVED*** private String getTemplatesDir() { +***REMOVED*** String input = null; +***REMOVED*** input = AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); +***REMOVED*** return input; +***REMOVED*** ***REMOVED*** +***REMOVED*** +***REMOVED*** +***REMOVED*** private String getCustomDir() { +***REMOVED*** String input = null; +***REMOVED*** input = AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); +***REMOVED*** return input; +***REMOVED*** ***REMOVED*** + + + + + - private File getCustomDir() { - return new File(this.dpmRoot, "custom"); - ***REMOVED*** public AnsibleWorker createWorker() { File workerRoot = new File(this.getWorkDir(), UUID.randomUUID().toString()); @@ -221,15 +233,15 @@ public class AnsibleBridge { private TemplateManager getTemplateManager() { - return new TemplateManager(this.dpmRoot+"/templates"); + return new TemplateManager(); ***REMOVED*** private CustomRoleManager getCustomRoleManager() { - return new CustomRoleManager(this.dpmRoot+"/custom"); + return new CustomRoleManager(); ***REMOVED*** private StaticRoleManager getStaticRoleManager() { - return new StaticRoleManager(this.dpmRoot+"/static"); + return new StaticRoleManager(); ***REMOVED*** /** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java index 8f583c0..b16805f 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/CustomRoleManager.java @@ -5,18 +5,21 @@ import java.io.File; import java.util.NoSuchElementException; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; +import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; public class CustomRoleManager { private String root; - public CustomRoleManager(String root) { - this.root = root; + public CustomRoleManager() { + ***REMOVED*** public String getRoot() { - return this.root; + String input = AnsibleBridge.class.getClassLoader().getResource("custom").getPath(); + + return input; ***REMOVED*** public Role getRole(String roleName) throws NoSuchElementException { diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java index 2dc24c1..ece553e 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/StaticRoleManager.java @@ -2,28 +2,30 @@ package org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.template; import java.io.File; ***REMOVED*** +import java.io.InputStream; import java.util.Collection; import java.util.Vector; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; +import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; public class StaticRoleManager { - private String root; - public StaticRoleManager(String root) { - this.root = root; + public StaticRoleManager() { + ***REMOVED*** public String getRoot() { - return this.root; + String input = AnsibleBridge.class.getClassLoader().getResource("static").getPath(); + return input; ***REMOVED*** public Collection getStaticRoles() { Collection out = new Vector<>(); for(File f: new File(this.getRoot()).listFiles()) { - try { + try { out.add(AnsibleSerializeHelper.deserializeRoleFromFilesystem(f)); ***REMOVED*** catch(IOException e) { e.printStackTrace(); diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java index 997b42a..bddb914 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/ansiblebridge/template/TemplateManager.java @@ -7,19 +7,19 @@ import java.util.NoSuchElementException; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.Role; import org.gcube.dataanalysis.dataminer.poolmanager.ansible.model.RoleFile; +import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleBridge; import org.gcube.dataanalysis.dataminer.poolmanager.ansiblebridge.AnsibleSerializeHelper; import org.stringtemplate.v4.ST; public class TemplateManager { - private String root; - - public TemplateManager(String root) { - this.root = root; + public TemplateManager() { + ***REMOVED*** public String getTemplateRoot() { - return this.root; + String input = AnsibleBridge.class.getClassLoader().getResource("templates").getPath(); + return input; ***REMOVED*** /** diff --git a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java index d65de22..a961278 100644 --- a/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java +++ b/src/main/java/org/gcube/dataanalysis/dataminer/poolmanager/service/DataminerPoolManager.java @@ -542,7 +542,7 @@ public class DataminerPoolManager implements PoolManager { public URL getURLfromWorkerLog(String a) throws MalformedURLException, UnknownHostException { - File path = new File(System.getProperty("user.home") + File.separator + "/gcube/dataminer-pool-manager/work/" + File path = new File(System.getProperty("user.home") + File.separator + "dataminer-pool-manager/work/" + a + File.separator + "logs"); path.mkdirs(); File n = new File(path + File.separator + a); diff --git a/src/main/resources/custom/custom-flr-Flash/meta/main.yaml b/src/main/resources/custom/custom-flr-Flash/meta/main.yaml new file mode 100644 index 0000000..b24c771 --- /dev/null +++ b/src/main/resources/custom/custom-flr-Flash/meta/main.yaml @@ -0,0 +1,4 @@ +#--- +#dependencies: +# +# - { role: digest ***REMOVED*** diff --git a/src/main/resources/custom/custom-flr-Flash/tasks/main.yaml b/src/main/resources/custom/custom-flr-Flash/tasks/main.yaml new file mode 100644 index 0000000..a7ee4e6 --- /dev/null +++ b/src/main/resources/custom/custom-flr-Flash/tasks/main.yaml @@ -0,0 +1,4 @@ +# tasks file for r +--- +- name: Install custom algorithm + command: Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('');" diff --git a/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml b/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml new file mode 100644 index 0000000..27a57fb --- /dev/null +++ b/src/main/resources/static/gcube-algorithm-installer/meta/main.yaml @@ -0,0 +1,4 @@ +dependencies: +# - { role: gcube-dataminer ***REMOVED*** +# - { role: os-unzip ***REMOVED*** +# - { role: os-java-1.7.0 ***REMOVED*** diff --git a/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml b/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml new file mode 100644 index 0000000..27c0e91 --- /dev/null +++ b/src/main/resources/static/gcube-algorithm-installer/tasks/main.yaml @@ -0,0 +1,31 @@ +--- + +- name: remove previous installer (if any) + file: + path: /home/gcube/algorithmInstaller + state: absent + +#- name: remove previous installer.zip (if any) +# file: +# path: /home/dpm/algorithmInstaller.zip +# state: absent + +#- name: download the installer zip +# get_url: +# url: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip +# dest: /home/dpm/algorithmInstaller.zip +# validate_certs: no + +- name: download and unzip the package + unarchive: + src: https:***REMOVED***svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/DataminerAlgorithmsInstaller/package/algorithmInstaller.zip + dest: /home/gcube + remote_src: yes + validate_certs: no + +- name: change flags + file: + path: /home/gcube/algorithmInstaller + mode: 0777 + state: directory + recurse: yes diff --git a/src/main/resources/static/gcube-dataminer/meta/main.yaml b/src/main/resources/static/gcube-dataminer/meta/main.yaml new file mode 100644 index 0000000..2f68d3c --- /dev/null +++ b/src/main/resources/static/gcube-dataminer/meta/main.yaml @@ -0,0 +1,2 @@ +dependencies: + - { role: gcube-ghn ***REMOVED*** diff --git a/src/main/resources/static/gcube-dataminer/tasks/main.yaml b/src/main/resources/static/gcube-dataminer/tasks/main.yaml new file mode 100644 index 0000000..628c691 --- /dev/null +++ b/src/main/resources/static/gcube-dataminer/tasks/main.yaml @@ -0,0 +1,4 @@ +--- + +- name: Install the 'DATAMINER' package + shell: echo 'installing DATAMINER' \ No newline at end of file diff --git a/src/main/resources/static/gcube-ghn/tasks/main.yaml b/src/main/resources/static/gcube-ghn/tasks/main.yaml new file mode 100644 index 0000000..9808bf4 --- /dev/null +++ b/src/main/resources/static/gcube-ghn/tasks/main.yaml @@ -0,0 +1,7 @@ +--- + +- name: Install the 'GHN' package + shell: echo 'installing GHN' + +# TODO: handler to start container +# TODO: handler to stop container \ No newline at end of file diff --git a/src/main/resources/static/os-java-1.7.0/tasks/main.yaml b/src/main/resources/static/os-java-1.7.0/tasks/main.yaml new file mode 100644 index 0000000..3acc5a3 --- /dev/null +++ b/src/main/resources/static/os-java-1.7.0/tasks/main.yaml @@ -0,0 +1,7 @@ +--- + +- name: adding OS Package java-1.7.0 + package: + name: java-1.7.0-openjdk + state: "{{os_package_state***REMOVED******REMOVED***" + become: true diff --git a/src/main/resources/static/os-r/tasks/main.yaml b/src/main/resources/static/os-r/tasks/main.yaml new file mode 100644 index 0000000..a8b40b2 --- /dev/null +++ b/src/main/resources/static/os-r/tasks/main.yaml @@ -0,0 +1 @@ +#to fill with R installation steps diff --git a/src/main/resources/static/os-unzip/tasks/main.yaml b/src/main/resources/static/os-unzip/tasks/main.yaml new file mode 100644 index 0000000..25d085d --- /dev/null +++ b/src/main/resources/static/os-unzip/tasks/main.yaml @@ -0,0 +1,7 @@ +--- + +- name: adding OS Package unzip + package: + name: unzip + state: "{{os_package_state***REMOVED******REMOVED***" + become: true diff --git a/src/main/resources/templates/cran-package-add/meta/main.yaml b/src/main/resources/templates/cran-package-add/meta/main.yaml new file mode 100644 index 0000000..dbe153f --- /dev/null +++ b/src/main/resources/templates/cran-package-add/meta/main.yaml @@ -0,0 +1,4 @@ +--- +#dependencies: +# +# - { role: os-r ***REMOVED*** diff --git a/src/main/resources/templates/cran-package-add/tasks/main.yaml b/src/main/resources/templates/cran-package-add/tasks/main.yaml new file mode 100644 index 0000000..42dd680 --- /dev/null +++ b/src/main/resources/templates/cran-package-add/tasks/main.yaml @@ -0,0 +1,4 @@ +# tasks file for r +--- +- name: Install algorithm + command: Rscript -e "install.packages('', repos=c('http:***REMOVED***ftp.heanet.ie/mirrors/cran.r-project.org/'))" diff --git a/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml b/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml new file mode 100644 index 0000000..b8f3340 --- /dev/null +++ b/src/main/resources/templates/gcube-algorithm-add/meta/main.yaml @@ -0,0 +1,4 @@ +--- +dependencies: + + - { role: gcube-algorithm-installer ***REMOVED*** diff --git a/src/main/resources/templates/gcube-algorithm-add/tasks/main.yaml b/src/main/resources/templates/gcube-algorithm-add/tasks/main.yaml new file mode 100644 index 0000000..44e52b0 --- /dev/null +++ b/src/main/resources/templates/gcube-algorithm-add/tasks/main.yaml @@ -0,0 +1,6 @@ +--- + +- name: Install algorithm + shell: /home/gcube/algorithmInstaller/addAlgorithm.sh "" + args: + chdir: /home/gcube/algorithmInstaller diff --git a/src/main/resources/templates/github-package-add/meta/main.yaml b/src/main/resources/templates/github-package-add/meta/main.yaml new file mode 100644 index 0000000..dbe153f --- /dev/null +++ b/src/main/resources/templates/github-package-add/meta/main.yaml @@ -0,0 +1,4 @@ +--- +#dependencies: +# +# - { role: os-r ***REMOVED*** diff --git a/src/main/resources/templates/github-package-add/tasks/main.yaml b/src/main/resources/templates/github-package-add/tasks/main.yaml new file mode 100644 index 0000000..5ccffa2 --- /dev/null +++ b/src/main/resources/templates/github-package-add/tasks/main.yaml @@ -0,0 +1,4 @@ +# tasks file for r +--- +- name: Install algorithm + command: Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('');" diff --git a/src/main/resources/templates/os-package-add/tasks/main.yaml b/src/main/resources/templates/os-package-add/tasks/main.yaml new file mode 100644 index 0000000..71a2609 --- /dev/null +++ b/src/main/resources/templates/os-package-add/tasks/main.yaml @@ -0,0 +1,8 @@ +--- + +- name: adding OS Package + package: + name: + state: "{{os_package_state***REMOVED******REMOVED***" + become: true +