From 2162ab2003b70016e65eafd62bbe2e8af7513ef6 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Thu, 25 Feb 2021 14:09:55 +0100 Subject: [PATCH] First commit. --- README.md | 151 +++++++++++++-- defaults/main.yml | 138 +++++++++++++- files/r_packages_cleanup.sh | 15 ++ meta/main.yml | 47 ++--- tasks/main.yml | 11 +- tasks/r-installation-deb.yml | 52 ++++++ tasks/r-installation-el.yml | 24 +++ tasks/r-packages-install.yml | 77 ++++++++ tasks/r-packages-updater.yml | 61 ++++++ tasks/r-packages_cleanup.yml | 8 + templates/Rprofile.site.j2 | 22 +++ templates/update_r_packages.sh.j2 | 298 ++++++++++++++++++++++++++++++ vars/main.yml | 2 +- 13 files changed, 852 insertions(+), 54 deletions(-) create mode 100644 files/r_packages_cleanup.sh create mode 100644 tasks/r-installation-deb.yml create mode 100644 tasks/r-installation-el.yml create mode 100644 tasks/r-packages-install.yml create mode 100644 tasks/r-packages-updater.yml create mode 100644 tasks/r-packages_cleanup.yml create mode 100644 templates/Rprofile.site.j2 create mode 100644 templates/update_r_packages.sh.j2 diff --git a/README.md b/README.md index 3637db8..7a65fb0 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,146 @@ Role Name ========= -A brief description of the role goes here. - -Requirements ------------- - -Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. +A role that installs the R environment, Role Variables -------------- -A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. +The most important variables are listed below: + +``` yaml +# Set to True if you want install from the CRAN deb repository +r_install_cran_repo: False +#r_cran_mirror_site: http://cran.rstudio.com +r_cran_set_default_mirror: True +r_cran_mirror_site: https://cran.mirror.garr.it/mirrors/CRAN/ +r_base_specific_version: False +r_base_pkg_version: 3.4.4 +r_packages_main_state: present +r_packages_state: '{{ r_packages_main_state }}' +r_sitelib_path: '/usr/local/lib/R/site-library' +r_plugins_from_deb: True +r_packages_cleanup: False +# +r_packages_updater: False +r_package_updater_via_subversion: True +r_package_updater_subversion_repo: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ +r_packages_svn_base_dir: /srv/r_updater +r_packages_svn_files_prefix: '' +r_packages_updater_also_upgrades_installed_packages: False +# They need to be flat text files +# 1 package per line +#r_debian_packages_list_url +# package[:cran mirror] +# The CRAN mirror URL is optional +#r_cran_packages_list_url +# The CRAN mirror URL is optional +#r_cran_packages_list_url +# user/package_name +#r_github_packages_list_url + +r_source_plugins_dest_dir: /var/cache/R + +r_distribution_required_packages: [] +# - gdal-bin +# - dans-gdal-scripts +# - libgdal1-dev +# - libgeos-dev +# - libspatialite-dev +# - proj +# - proj-bin +# - proj-data +# - libproj-dev + +r_base_packages_list: + - r-base + +# Some packages must be put on hold +r_base_packages_hold_list: [] + +r_plugins_packages_list: + - jags + - r-cran-rjags + - r-cran-abind + - r-cran-boot + - r-cran-class + - r-cran-cluster + - r-cran-coda + - r-cran-codetools + - r-cran-foreign + - r-cran-lattice + - r-cran-maptools + - r-cran-mass + - r-cran-matrix + - r-cran-mgcv + - r-cran-nlme + - r-cran-nnet + - r-cran-rpart + - r-cran-sp + - r-cran-spatial + - r-cran-survival + +r_apt_additional_repos: + - 'ppa:opencpu/jq' + +# RHEL/CentOS: +# http://springdale.princeton.edu/data/springdale/7/x86_64/os/Computational/ +# +r_sitelib_el_path: '/usr/local/lib/R/library' +r_el_dev_pkgs: True +r_base_packages_el: + - R + - R-core + - libRmath + +r_base_packages_dev_el: + - R-core-devel + - libRmath-devel + +r_base_packages_java_el: + - R-java + +r_base_packages_java_dev_el: + - R-java-devel + +r_plugins_list_to_install: [] +# - { name: 'R2WinBUGS', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'R2jags', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'bayesmix', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'coda', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'rjags', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'runjags', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'base', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'compiler', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'datasets', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'grDevices', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'graphics', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'grid', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'methods', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'parallel', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'reshape', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'splines', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'stats', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'stats4', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'tcltk', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'tools', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'utils', repo: '{{ r_cran_mirror_site }}' } + +r_plugins_from_sources: [] +# - { name: 'plyr', version: '1.7.1', source: 'plyr_1.7.1.tar.gz', url: 'http://cran.r-project.org/src/contrib/Archive/plyr/plyr_1.7.1.tar.gz' } + +r_plugins_from_github: [] +# - { plugin_name: 'RFigisGeo', github_user: 'openfigis' } +# - { plugin_name: 'rsdmx', github_user: 'opensdmx' } + +# +r_plugins_list_to_remove: [] +``` Dependencies ------------ -A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. - -Example Playbook ----------------- - -Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: - - - hosts: servers - roles: - - { role: username.rolename, x: 42 } +None License ------- @@ -35,4 +150,4 @@ EUPL-1.2 Author Information ------------------ -An optional section for the role authors to include contact information, or a website (HTML is not allowed). +Andrea Dell'Amico, diff --git a/defaults/main.yml b/defaults/main.yml index 95d3c70..bc49073 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,2 +1,138 @@ --- -# defaults file for ansible-role-template \ No newline at end of file +# +# There's an untrusted Ubuntu PPA repository with gobs of packages and dependencies: https://launchpad.net/~marutter/+archive/ubuntu/c2d4u +# +# To list the installed R packages +# Run R, then execute +# packinfo <- installed.packages (fields = c ("Package", "Version")) +# packinfo[,c("Package", "Version")] +# +# The install/remove script has been taken from here: http://adamj.eu/tech/2014/07/19/installing-and-removing-r-packages-with-ansible/ +# +# Set to True if you want install from the CRAN deb repository +r_install_cran_repo: False +#r_cran_mirror_site: http://cran.rstudio.com +r_cran_set_default_mirror: True +r_cran_mirror_site: https://cran.mirror.garr.it/mirrors/CRAN/ +r_base_specific_version: False +r_base_pkg_version: 3.4.4 +r_packages_main_state: present +r_packages_state: '{{ r_packages_main_state }}' +r_sitelib_path: '/usr/local/lib/R/site-library' +r_plugins_from_deb: True +r_packages_cleanup: False +# +r_packages_updater: False +r_package_updater_via_subversion: True +r_package_updater_subversion_repo: http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ +r_packages_svn_base_dir: /srv/r_updater +r_packages_svn_files_prefix: '' +r_packages_updater_also_upgrades_installed_packages: False +# They need to be flat text files +# 1 package per line +#r_debian_packages_list_url +#r_el_packages_list_url +# package[:cran mirror] +# The CRAN mirror URL is optional +#r_cran_packages_list_url +# The CRAN mirror URL is optional +#r_cran_packages_list_url +# user/package_name +#r_github_packages_list_url + +r_source_plugins_dest_dir: /var/cache/R + +r_distribution_required_packages: [] +# - gdal-bin +# - dans-gdal-scripts +# - libgdal1-dev +# - libgeos-dev +# - libspatialite-dev +# - proj +# - proj-bin +# - proj-data +# - libproj-dev + +r_base_packages_list: + - r-base + +# Some packages must be put on hold +r_base_packages_hold_list: [] + +r_plugins_packages_list: + - jags + - r-cran-rjags + - r-cran-abind + - r-cran-boot + - r-cran-class + - r-cran-cluster + - r-cran-coda + - r-cran-codetools + - r-cran-foreign + - r-cran-lattice + - r-cran-maptools + - r-cran-mass + - r-cran-matrix + - r-cran-mgcv + - r-cran-nlme + - r-cran-nnet + - r-cran-rpart + - r-cran-sp + - r-cran-spatial + - r-cran-survival + +r_apt_additional_repos: + - 'ppa:opencpu/jq' + +# RHEL/CentOS: +# http://springdale.princeton.edu/data/springdale/7/x86_64/os/Computational/ +# +r_sitelib_el_path: '/usr/local/lib/R/library' +r_el_dev_pkgs: True +r_base_packages_el: + - R + - R-core + - libRmath + +r_base_packages_dev_el: + - R-core-devel + - libRmath-devel + +r_base_packages_java_el: + - R-java + +r_base_packages_java_dev_el: + - R-java-devel + +r_plugins_list_to_install: [] +# - { name: 'R2WinBUGS', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'R2jags', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'bayesmix', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'coda', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'rjags', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'runjags', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'base', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'compiler', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'datasets', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'grDevices', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'graphics', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'grid', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'methods', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'parallel', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'reshape', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'splines', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'stats', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'stats4', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'tcltk', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'tools', repo: '{{ r_cran_mirror_site }}' } +# - { name: 'utils', repo: '{{ r_cran_mirror_site }}' } + +r_plugins_from_sources: [] +# - { name: 'plyr', version: '1.7.1', source: 'plyr_1.7.1.tar.gz', url: 'http://cran.r-project.org/src/contrib/Archive/plyr/plyr_1.7.1.tar.gz' } + +r_plugins_from_github: [] +# - { plugin_name: 'RFigisGeo', github_user: 'openfigis' } +# - { plugin_name: 'rsdmx', github_user: 'opensdmx' } + +# +r_plugins_list_to_remove: [] diff --git a/files/r_packages_cleanup.sh b/files/r_packages_cleanup.sh new file mode 100644 index 0000000..c506d40 --- /dev/null +++ b/files/r_packages_cleanup.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Put the base packages to unhold +for package in r-base r-base-core r-base-dev r-base-html r-cran-boot r-cran-class r-cran-cluster r-cran-codetools r-cran-foreign r-cran-kernsmooth r-cran-lattice r-cran-mass r-cran-matrix r-cran-mgcv r-cran-nlme r-cran-nnet r-cran-rpart r-cran-spatial r-cran-survival r-doc-html r-recommended ; do apt-mark unhold $package; done + +# Remove the old r packages +apt-get purge r-base-* r-cran-* -y --force-yes +apt-get autoremove -y +apt-get update + +# Remove the CRAN packages +rm -fr /usr/lib/R/site-library /usr/local/lib/R/site-library +rm -fr /var/cache/R/* + +exit 0 diff --git a/meta/main.yml b/meta/main.yml index 81bda14..e70057b 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,46 +1,27 @@ galaxy_info: - author: your name - description: your description - company: your company (optional) + author: Andrea Dell'Amico + description: Systems Architect + company: ISTI-CNR - # If the issue tracker for your role is not on github, uncomment the - # next line and provide a value - issue_tracker_url: https://support.d4science.org/projects/automatic-provisioning/issues + issue_tracker_url: https://redmine-s2i2s.isti.cnr.it/projects/provisioning - license: EUPL-1.2 + license: EUPL 1.2+ min_ansible_version: 2.8 - # If this a Container Enabled role, provide the minimum Ansible Container version. - # min_ansible_container_version: - - # Optionally specify the branch Galaxy will use when accessing the GitHub - # repo for this role. During role install, if no tags are available, - # Galaxy will use this branch. During import Galaxy will access files on - # this branch. If Travis integration is configured, only notifications for this - # branch will be accepted. Otherwise, in all cases, the repo's default branch - # (usually master) will be used. - #github_branch: - - # - # Provide a list of supported platforms, and for each platform a list of versions. - # If you don't wish to enumerate all versions for a particular platform, use 'all'. # To view available platforms and versions (or releases), visit: # https://galaxy.ansible.com/api/v1/platforms/ # platforms: - - name: Ubuntu - versions: - - bionic + - name: Ubuntu + versions: + - bionic + - name: EL + versions: + - 7 - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes - # and categorizes the role. Users find roles by searching for tags. Be sure to - # remove the '[]' above, if you add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. + galaxy_tags: + - R + - CRAN dependencies: [] - # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. \ No newline at end of file diff --git a/tasks/main.yml b/tasks/main.yml index 53c6cae..126a476 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,2 +1,11 @@ --- -# tasks file for ansible-role-template \ No newline at end of file +- import_tasks: r-packages_cleanup.yml + when: + - r_packages_cleanup + - ansible_distribution_file_variety == "Debian" +- import_tasks: r-installation-deb.yml + when: ansible_distribution_file_variety == "Debian" +- import_tasks: r-installation-el.yml + when: ansible_distribution_file_variety == "RedHat" +- import_tasks: r-packages-install.yml +- import_tasks: r-packages-updater.yml diff --git a/tasks/r-installation-deb.yml b/tasks/r-installation-deb.yml new file mode 100644 index 0000000..f231149 --- /dev/null +++ b/tasks/r-installation-deb.yml @@ -0,0 +1,52 @@ +--- +- block: + - name: Add the cran repository key + apt_key: id=E084DAB9 keyserver=keyserver.ubuntu.com state=present + tags: [ 'r_software', 'r_repo', 'r_repo_key' ] + + - name: Add the CRAN repository + apt_repository: repo='deb http://cran.rstudio.com/bin/linux/ubuntu {{ ansible_distribution_release }}/' state=present update_cache=yes + + when: r_install_cran_repo + tags: [ 'r_software', 'r_repo' ] + +- block: + - name: Remove the CRAN repository + apt_repository: repo='deb http://cran.rstudio.com/bin/linux/ubuntu {{ ansible_distribution_release }}/' state=absent update_cache=yes + + when: not r_install_cran_repo + tags: [ 'r_software', 'r_repo' ] + +- block: + - name: Remove the hold state from the debian R packages + shell: apt-mark unhold {{ item }} + with_items: '{{ r_base_packages_hold_list | default([]) }}' + when: r_base_packages_hold_list is defined + ignore_errors: True + tags: [ 'r_software', 'r_pkg', 'r_pkg_hold' ] + + - name: Install the R base packages. + apt: pkg={{ r_base_packages_list }} state={{ r_packages_main_state }} force=yes update_cache=yes cache_valid_time=3600 + + - name: When we install specific R deb packages, put them on hold + shell: apt-mark hold {{ item }} + with_items: '{{ r_base_packages_hold_list| default([]) }}' + when: r_base_specific_version + ignore_errors: True + tags: [ 'r_software', 'r_pkg', 'r_pkg_hold' ] + + - name: Install the R additional modules from the deb repo + apt: pkg={{ r_plugins_packages_list | default([]) }} state={{ r_packages_state }} force=yes + when: r_plugins_from_deb + + - name: Install some additional repositories. They provide dependencies for some R packages + apt_repository: repo={{ item }} state=present update_cache=yes + with_items: '{{ r_apt_additional_repos }}' + when: ansible_distribution_version is version_compare('18.04', '<') + tags: [ 'r_software', 'r_apt_repo', 'r_deps' ] + + - name: Install some packages needed by R packages when installed from source + apt: pkg={{ r_distribution_required_packages | default([]) }} state={{ r_packages_state }} update_cache=yes force=yes cache_valid_time=3600 + tags: [ 'r_software', 'r_pkg', 'r_deps' ] + + tags: [ 'r_software', 'r_pkg' ] diff --git a/tasks/r-installation-el.yml b/tasks/r-installation-el.yml new file mode 100644 index 0000000..cbbde9e --- /dev/null +++ b/tasks/r-installation-el.yml @@ -0,0 +1,24 @@ +--- +- block: + - name: Install the R base packages. + yum: pkg={{ r_base_packages_el }} state={{ r_packages_main_state }} + + - name: Install the R base development packages. + yum: pkg={{ r_base_packages_dev_el }} state={{ r_packages_main_state }} + when: r_el_dev_pkgs + + - name: Install the R java base packages. + yum: pkg={{ r_base_packages_java_el }} state={{ r_packages_main_state }} + when: jdk_java_home is defined + + - name: Install the R java base development packages. + yum: pkg={{ r_base_packages_java_dev_el }} state={{ r_packages_main_state }} + when: + - jdk_java_home is defined + - r_el_dev_pkgs + + - name: Install some packages needed by R packages when installed from source + yum: pkg={{ r_distribution_required_el_packages }} state={{ r_packages_state }} + tags: [ 'r_software', 'r_pkg', 'r_deps' ] + + tags: [ 'r_software', 'r_pkg' ] diff --git a/tasks/r-packages-install.yml b/tasks/r-packages-install.yml new file mode 100644 index 0000000..baa3ea4 --- /dev/null +++ b/tasks/r-packages-install.yml @@ -0,0 +1,77 @@ +--- +- block: + - name: Configure the default CRAN mirror + template: src=Rprofile.site.j2 dest=/etc/R/Rprofile.site owner=root group=root mode=0444 + when: r_install_cran_repo == 'present' + tags: [ 'r_software', 'r_profile', 'r_pkg' ] + + - name: Configure the JDK environment + shell: export JAVA_HOME={{ jdk_java_home }} ; export J2SDKDIR={{ jdk_java_home }} ; export J2REDIR={{ jdk_java_home }}/jre ; R CMD javareconf ; touch /etc/R/.java{{ jdk_default }}.env_conf + args: + creates: '/etc/R/.java{{ jdk_default }}.env_conf' + when: + - jdk_java_home is defined + - jdk_default is defined + tags: [ 'r_software', 'r_profile', 'r_pkg', 'r_java' ] + + - name: Ensure that the R packages sources directory exists + file: dest={{ r_source_plugins_dest_dir }} state=directory owner=root group=root + when: r_plugins_from_sources is defined + tags: [ 'r_software', 'r_pkg', 'r_plugins', 'r_plugins_version' ] + +- block: + - name: Install R packages from the cran sources repo or from an alternative repository, latest available version. First try + command: > + Rscript --slave --no-save --no-restore-history -e "if (! ('{{ item.name }}' %in% installed.packages()[,'Package'])) { install.packages(pkgs='{{ item.name }}', repos=c('{{ item.repo | default ('https://cloud.r-project.org') }}/')); print('Added'); } else { print('Already installed'); }" + register: install_plugins_result + failed_when: "install_plugins_result.rc != 0 or 'had non-zero exit status' in install_plugins_result.stderr" + changed_when: "'Added' in install_plugins_result.stdout" + with_items: '{{ r_plugins_list_to_install | default([]) }}' + ignore_errors: True + + - name: Get the R packages sources that need to be installed + get_url: url={{ item.url }} dest={{ r_source_plugins_dest_dir }} + with_items: '{{ r_plugins_from_sources | default([]) }}' + tags: [ 'r_software', 'r_pkg', 'r_plugins', 'r_plugins_version' ] + + - name: Install R packages from the cran sources, specific versions. First round + command: > + Rscript --slave --no-save --no-restore-history -e "if (! ('{{ item.name }}' %in% installed.packages()[,'Package'])) { install.packages('{{ r_source_plugins_dest_dir }}/{{ item.source }}', repos = NULL, type='source'); print('Added'); } else if (packageVersion('{{ item.name }}') != '{{ item.version }}') { install.packages('{{ r_source_plugins_dest_dir }}/{{ item.source }}', repos = NULL, type='source'); print('Added'); } else { print('Already Installed'); }" + register: install_s_plugins_result + failed_when: "install_s_plugins_result.rc != 0 or 'had non-zero exit status' in install_s_plugins_result.stderr" + changed_when: '"Added" in install_s_plugins_result.stdout' + with_items: '{{ r_plugins_from_sources | default([]) }}' + ignore_errors: True + tags: [ 'r_software', 'r_pkg', 'r_plugins', 'r_plugins_version' ] + + - name: Install R packages from github + command: > + Rscript --slave --no-save --no-restore-history -e "if (! ('{{ item.plugin_name }}' %in% installed.packages()[,'Package'])) { require(devtools); require(methods) ; options(repos='{{ r_cran_mirror_site }}/') ; install_github('{{ item.github_user }}/{{ item.plugin_name }}'); print('Added'); } else { print('Already Installed'); }" + register: install_github_plugins_result + failed_when: "install_github_plugins_result.rc != 0 or 'had non-zero exit status' in install_github_plugins_result.stderr" + changed_when: "'Added' in install_github_plugins_result.stdout" + with_items: '{{ r_plugins_from_github | default([]) }}' + tags: [ 'r_software', 'r_pkg', 'r_plugins', 'r_plugins_github' ] + ignore_errors: True + + - name: Install R packages from the cran sources, specific versions. + command: > + Rscript --slave --no-save --no-restore-history -e "if (! ('{{ item.name }}' %in% installed.packages()[,'Package'])) { install.packages('{{ r_source_plugins_dest_dir }}/{{ item.source }}', repos = NULL, type='source'); print('Added'); } else if (packageVersion('{{ item.name }}') != '{{ item.version }}') { install.packages('{{ r_source_plugins_dest_dir }}/{{ item.source }}', repos = NULL, type='source'); print('Added'); } else { print('Already Installed'); }" + register: install_v_plugins_result + failed_when: "install_v_plugins_result.rc != 0 or 'had non-zero exit status' in install_v_plugins_result.stderr" + changed_when: '"Added" in install_s_plugins_result.stdout' + with_items: '{{ r_plugins_from_sources | default([]) }}' + ignore_errors: True + tags: [ 'r_software', 'r_pkg', 'r_plugins', 'r_plugins_version' ] + + - name: Remove R unwanted packages + command: > + Rscript --slave --no-save --no-restore-history -e "if (! ('{{ item }}' %in% installed.packages()[,'Package'])) { print('Not installed'); } else { remove.packages(pkgs='{{ item }}'); print('Removed'); }" + register: remove_plugins_result + failed_when: remove_plugins_result.rc != 0 + changed_when: "'Removed' in remove_plugins_result.stdout" + with_items: '{{ r_plugins_list_to_remove | default([]) }}' + when: r_plugins_list_to_remove is defined + + tags: [ 'r_software', 'r_pkg', 'r_plugins' ] + diff --git a/tasks/r-packages-updater.yml b/tasks/r-packages-updater.yml new file mode 100644 index 0000000..cbee07f --- /dev/null +++ b/tasks/r-packages-updater.yml @@ -0,0 +1,61 @@ +--- +- block: + - name: Install the R packages updater script + template: + src: 'update_r_packages.sh.j2' + dest: '/usr/local/bin/update_r_packages' + owner: root + group: root + mode: '0755' + + - name: Create the R packages updater SVN base directory + file: + dest: '{{ r_packages_svn_base_dir }}' + state: directory + + - name: Cron job that installs new R packages, if any + cron: + name: "install new R packages" + user: root + cron_file: install-r-packages + minute: "*/10" + hour: "5-23,1-2" + job: "/usr/local/bin/update_r_packages install >/var/log/install_r_packages 2>&1" + state: present + + - name: Cron job that upgrades existing R packages and installs new ones, if any + cron: + name: "Upgrade installed R packages" + user: root + cron_file: upgrade-r-packages + minute: "7" + hour: "3" + job: "/usr/local/bin/update_r_packages upgrade >/var/log/update_r_packages 2>&1" + state: present + when: r_packages_updater_also_upgrades_installed_packages + + when: r_packages_updater + tags: [ 'r_software', 'r_pkg', 'r_plugins', 'r_plugins_github', 'r_cran_pkgs', 'r_github_pkgs', 'r_packages_updater' ] + +- block: + - name: Remove the R packages updater script + file: dest=/usr/local/bin/update_r_packages state=absent + + - name: Remove the R packages updater SVN base directory + file: dest={{ r_packages_svn_base_dir }} state=absent + + - name: Remove the cron job that installs new R packages + file: dest=/etc/cron.d/install-r-packages state=absent + + - name: Remove the cron job that upgrades existing R packages and installs new ones + file: dest=/etc/cron.d/upgrade-r-packages state=absent + + when: not r_packages_updater + tags: [ 'r_software', 'r_pkg', 'r_plugins', 'r_plugins_github', 'r_cran_pkgs', 'r_github_pkgs', 'r_packages_updater' ] + +- block: + - name: Disable the cron job that upgrades already installed R packages + file: dest=/etc/cron.d/upgrade-r-packages state=absent + + when: not r_packages_updater_also_upgrades_installed_packages + tags: [ 'r_software', 'r_pkg', 'r_plugins', 'r_plugins_github', 'r_cran_pkgs', 'r_github_pkgs', 'r_packages_updater' ] diff --git a/tasks/r-packages_cleanup.yml b/tasks/r-packages_cleanup.yml new file mode 100644 index 0000000..76e8720 --- /dev/null +++ b/tasks/r-packages_cleanup.yml @@ -0,0 +1,8 @@ +--- +- name: Install the script that cleans up the R deb and cran packages + copy: src=r_packages_cleanup.sh dest=/usr/local/bin/r_packages_cleanup owner=root group=root mode=0500 + tags: [ 'r_software', 'r_pkgs', 'r_cleanup' ] + +- name: Remove all the old R deb and cran packages. Otherwise the upgrade will fail miserably + shell: /usr/local/bin/r_packages_cleanup + tags: [ 'r_software', 'r_pkgs', 'r_cleanup' ] diff --git a/templates/Rprofile.site.j2 b/templates/Rprofile.site.j2 new file mode 100644 index 0000000..b2124d0 --- /dev/null +++ b/templates/Rprofile.site.j2 @@ -0,0 +1,22 @@ +## Emacs please make this -*- R -*- +## empty Rprofile.site for R on Debian +## +## Copyright (C) 2008 Dirk Eddelbuettel and GPL'ed +## +## see help(Startup) for documentation on ~/.Rprofile and Rprofile.site +# +# NOTE: managed by ansible +# +local({r <- getOption("repos") + r["CRAN"] <- "{{ r_cran_mirror_site }}" + options(repos=r) +}) + +# Explicitly set the R encoding using the system one, if it exists +if (file.exists("/etc/default/locale")) { + readRenviron("/etc/default/locale") + LANG <- Sys.getenv("LANG") + if(nchar(LANG)) + Sys.setlocale("LC_ALL", LANG) +} + diff --git a/templates/update_r_packages.sh.j2 b/templates/update_r_packages.sh.j2 new file mode 100644 index 0000000..acac568 --- /dev/null +++ b/templates/update_r_packages.sh.j2 @@ -0,0 +1,298 @@ +#!/bin/bash +# +# TODO: kill an old process if it is running from too much time (12 hours?) +# using something like ps -o etimes= -p "$PROCNUM" +# + +export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" + +RETVAl= +PARAMS=$# +ACTION=$1 +PROCNUM=$$ +OLDPROC= +OLDPROC_RUNNING= +LOCKDIR=/var/run +LOCK_FILE=$LOCKDIR/.update_r_pkgs.lock +TMP_FILES_DIR=/var/tmp/r_pkgs_update +# We cannot answer questions +DEBIAN_FRONTEND=noninteractive +R_CRAN_MIRROR="{{ r_cran_mirror_site }}" +R_PKGS_FROM_SVN="{{ r_package_updater_via_subversion }}" +R_PKGS_SVN_DIR=RPackagesManagement +R_PKGS_SVN_URL="{{ r_package_updater_subversion_repo }}" +R_PKGS_SVN_BASE_DIR="{{ r_packages_svn_base_dir }}" +R_PKGS_FILES_PREFIX="{{ r_packages_svn_files_prefix }}" +RSCRIPT_OPTIONS="--slave --no-site-file --no-init-file --no-save --no-restore-history" +SVN_UPDATE_STATUS= +# In seconds. 60*60*6=21600s (6h) +UPDATER_PROCESS_MAX_RUNTIME=21600 +# - debian packages list format: +# one package per line +DEB_PKGS_SKIP=0 +DEBIAN_PKGS_LIST_URL="{{ r_debian_packages_list_url | default('') }}" +EL_PKGS_LIST_URL="{{ r_el_packages_list_url | default('') }}" +PKGS_LIST= +# - R packages list format: +# name[:mirror] +CRAN_PKGS_SKIP=0 +R_PKGS_LIST_URL="{{ r_cran_packages_list_url | default('') }}" +R_PKGS_LIST= +# - R versioned packages list format: +# name:version +CRAN_VERSIONED_PKGS_SKIP=0 +R_VERSIONED_PKGS_LIST_URL="{{ r_cran_versioned_packages_list_url | default('') }}" +R_VERSIONED_PKGS_LIST= +# - R packages from github list format: +# - owner/package +GITHUB_PKGS_SKIP=0 +R_PKGS_FROM_GITHUB_LIST_URL="{{ r_github_packages_list_url | default('') }}" +R_PKGS_GITHUB= + +trap "logger 'update_r_packages: trap intercepted, exiting.' ; cleanup" SIGHUP SIGINT SIGTERM + +function cleanup() { + logger "update_r_packages: cleaning up" + rm -f $LOCK_FILE + rm -fr $TMP_FILES_DIR +} + +function usage() { + if [ $PARAMS -ne 1 ] ; then + echo "Need at least an argument: 'upgrade' or 'install'." + echo "- 'upgrade' installs new packages and upgrades the existin ones when needed." + echo "- 'install' installs new packages." + cleanup + exit 1 + fi +} + +function get_args() { + if [ "$ACTION" != "upgrade" -a "$ACTION" != "install" ] ; then + usage + fi +} + +function fail() { + logger "update_r_packages: Something went wrong, exiting." + cleanup + exit 1 +} + +function init_env() { + if [ -f $LOCK_FILE ] ; then + OLDPROC=$( cat $LOCK_FILE ) + OLDPROC_RUNNING=$( ps auwwx | grep -v grep | grep $OLDPROC | awk '{ print $2 }' ) + RETVAL=$? + if [ ! -z "$OLDPROC_RUNNING" ] ; then + logger "update_r_packages: pid of the already running process: $OLDPROC_RUNNING" + OLDPROC_RUNNING_TIME=$( ps -o etimes= -p ${OLDPROC_RUNNING} ) + if [ $OLDPROC_RUNNING_TIME -gt $UPDATER_PROCESS_MAX_RUNTIME ] ; then + logger "update_r_packages: process $OLDPROC_RUNNING was running for $OLDPROC_RUNNING_TIME seconds. Got stuck, killing it" + kill -9 $OLDPROC_RUNNING + cleanup + else + logger "update_r_packages: another process is running, exiting." + exit 0 + fi + else + logger "update_r_packages: lock file exist but the process not. Continuing." + rm -fr $TMP_FILES_DIR + fi + else + logger 'update_r_packages: no other jobs running, proceeding.' + fi + RETVAL= + echo "$PROCNUM" > $LOCK_FILE + mkdir -p $TMP_FILES_DIR +} + +function get_data_files() { + logger "update_r_packages: get the single files from http." + # Get the packages list + if [ -z $DEBIAN_PKGS_LIST_URL ] ; then + DEB_PKGS_SKIP=1 + logger "update_r_packages: the debian packages list is not available." + else + PKGS_LIST=$( mktemp $TMP_FILES_DIR/rdebs.XXXXXXX ) + logger "update_r_packages: getting the debian packages list." + wget -q -o /dev/null -O $PKGS_LIST $DEBIAN_PKGS_LIST_URL + fi + if [ -z $R_PKGS_LIST_URL ] ; then + CRAN_PKGS_SKIP=1 + logger "update_r_packages: the CRAN packages list is not available." + else + R_PKGS_LIST=$( mktemp $TMP_FILES_DIR/rpkgs.XXXXXXX ) + logger "update_r_packages: getting the R packages list that will be installed from CRAN" + wget -q -o /dev/null -O $R_PKGS_LIST $R_PKGS_LIST_URL + fi + if [ -z $R_VERSIONED_PKGS_LIST_URL ] ; then + CRAN_VERSIONED_PKGS_SKIP=1 + logger "update_r_packages: the CRAN versioned packages list is not available." + else + R_PKGS_LIST=$( mktemp $TMP_FILES_DIR/rpkgs.XXXXXXX ) + logger "update_r_packages: getting the R versioned packages list that will be installed from CRAN" + wget -q -o /dev/null -O $R_VERSIONED_PKGS_LIST $R_VERSIONED_PKGS_LIST_URL + fi + if [ -z $R_PKGS_FROM_GITHUB_LIST_URL ] ; then + GITHUB_PKGS_SKIP=1 + logger "update_r_packages: the Github packages list is not available." + else + R_PKGS_GITHUB=$( mktemp $TMP_FILES_DIR/rpkgsgithub.XXXXXXX ) + logger "update_r_packages: getting the R packages list that will be installed from github" + wget -q -o /dev/null -O $R_PKGS_GITHUB $R_PKGS_FROM_GITHUB_LIST_URL + fi +} + +function get_data_files_from_svn() { + logger "update_r_packages: files from a SVN repo." + if [ -d $R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR ] ; then + logger "update_r_packages: SVN update" + cd $R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR + SVN_CLEANUP_OP=$( svn cleanup ) + SVN_UPDATE_OP=$( svn update | tail -1 | grep Updated >/dev/null 2>&1 ) + SVN_UPDATE_STATUS=$? + else + cd $R_PKGS_SVN_BASE_DIR + logger "update_r_packages: first SVN checkout." + svn co $R_PKGS_SVN_URL >/dev/null 2>&1 + fi + PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_deb_pkgs.txt + R_PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_cran_pkgs.txt + R_VERSIONED_PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_cran_versioned_pkgs.txt + R_PKGS_GITHUB=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_github_pkgs.txt +} + +{% if ansible_distribution_file_variety == "Debian" %} +function distribution_pkgs() { + if [ $DEB_PKGS_SKIP -eq 0 ] ; then + # Update the apt cache and install the packages in non interactive mode + logger "update_r_packages: Installing the debian dependencies" + if [ -z "$(find /var/cache/apt/pkgcache.bin -mmin -360)" ]; then + apt-get update -q >/dev/null 2>&1 + else + logger "update_r_packages: APT cache not updated" + fi + >/var/log/update_r_debs.log + while read deb_pkg ; do + apt-get install ${deb_pkg} -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" >>/var/log/update_r_debs.log 2>&1 + done < $PKGS_LIST + apt-get autoremove -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" >> /var/log/update_r_debs.log 2>&1 + else + logger "update_r_packages: skipping the debian packages installation" + fi +} + +function remove_r_install_packages_lock_files() { + # install.packages leaves lock files around if the process crashes + rm -fr {{ r_sitelib_path }}/00LOCK-* + } +{% endif %} + +{% if ansible_distribution_file_variety == "RedHat" %} +function distribution_pkgs() { + if [ $EL_PKGS_SKIP -eq 0 ] ; then + # Update the apt cache and install the packages in non interactive mode + logger "update_r_packages: Installing the EL dependencies" + if [ -z "$(find /var/cache/apt/pkgcache.bin -mmin -360)" ]; then + apt-get update -q >/dev/null 2>&1 + else + logger "update_r_packages: APT cache not updated" + fi + >/var/log/update_r_debs.log + while read el_pkg ; do + yum -y install ${el_pkg} >>/var/log/update_r_el.log 2>&1 + done < $PKGS_LIST + else + logger "update_r_packages: skipping the EL packages installation" + fi +} + +function remove_r_install_packages_lock_files() { + # install.packages leaves lock files around if the process crashes + rm -fr {{ r_sitelib_path }}/00LOCK-* + } +{% endif %} + +function r_cran_pkgs() { + if [ $CRAN_PKGS_SKIP -eq 0 ] ; then + logger "update_r_packages: Installing R packages from CRAN" + for l in $( cat $R_PKGS_LIST ) ; do + pkg=$( echo $l | cut -d : -f 1 ) + is_mirror_ret= + is_mirror=$( echo $l | grep ':' ) + is_mirror_ret=$? + if [ $is_mirror_ret -eq 0 ] ; then + mirror=$( echo $l | cut -d : -f 2- ) + else + mirror=$R_CRAN_MIRROR + fi + if [ "$ACTION" == "upgrade" ] ; then + Rscript $RSCRIPT_OPTIONS -e "install.packages(pkgs='$pkg', repos=c('$mirror/'));" + else + Rscript $RSCRIPT_OPTIONS -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { install.packages(pkgs='$pkg', repos=c('$mirror/')); }" + fi + done + else + logger "update_r_packages: skipping the R CRAN packages installation" + fi +} + +function r_cran_versioned_pkgs() { + if [ $CRAN_VERSIONED_PKGS_SKIP -eq 0 ] ; then + logger "update_r_packages: Installing versioned R packages from CRAN" + for l in $( cat $R_VERSIONED_PKGS_LIST ) ; do + pkg=$( echo $l | cut -d : -f 1 ) + version=$( echo $l | cut -d : -f 2 ) + if [ "$ACTION" == "upgrade" ] ; then + Rscript $RSCRIPT_OPTIONS -e "require(devtools); install_version('$pkg', '$version', repos=c('$mirror/'));" + else + Rscript $RSCRIPT_OPTIONS -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { require(devtools); install_version('$pkg', '$version', repos=c('$mirror/')); }" + fi + done + else + logger "update_r_packages: skipping the R CRAN versioned packages installation" + fi +} + +function r_github_pkgs() { + if [ $GITHUB_PKGS_SKIP -eq 0 ] ; then + logger "update_r_packages: Installing R packages from Github" + for l in $( cat $R_PKGS_GITHUB ) ; do + pkg=$( echo $l | cut -d "/" -f 2 ) + if [ "$ACTION" == "upgrade" ] ; then + Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); require(jsonlite) ; package_to_install <- '$l' ; refs <- jsonlite::read_json(sprintf('https://api.github.com/repos/%s/releases', package_to_install)) ; ref_to_install <- 'master'; if(length(refs)>0) { ref_to_install <- refs[[1]][['tag_name']] } ; devtools::install_github(package_to_install, ref = ref_to_install, upgrade='always')" + + else + Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { require(devtools); require(methods); require(jsonlite) ; package_to_install <- '$l' ; refs <- jsonlite::read_json(sprintf('https://api.github.com/repos/%s/releases', package_to_install)) ; ref_to_install <- 'master'; if(length(refs)>0) { ref_to_install <- refs[[1]][['tag_name']] } ; devtools::install_github(package_to_install, ref = ref_to_install, upgrade='always') }" + fi + done + else + logger "update_r_packages: skipping the R GitHub packages installation" + fi +} + +######### +# Main +# + +usage +get_args +init_env +if [ $R_PKGS_FROM_SVN == 'True' ] ; then + get_data_files_from_svn + if [ $SVN_UPDATE_STATUS -ne 0 -a "$ACTION" == "install" ] ; then + logger "update_r_packages: nothing new to install from SVN, exiting" + cleanup + exit 0 + fi +else + get_data_files +fi +distribution_pkgs +remove_r_install_packages_lock_files +r_cran_pkgs +r_cran_versioned_pkgs +r_github_pkgs +cleanup +exit 0 diff --git a/vars/main.yml b/vars/main.yml index 3808477..3913f11 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -1,2 +1,2 @@ --- -# vars file for ansible-role-template \ No newline at end of file +centos_install_epel: true