From 52eb4dd1b077e4c1e1ebaf64e74b2a6b8bf993d3 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Wed, 3 Mar 2021 13:23:49 +0100 Subject: [PATCH] First commit. --- README.md | 61 +++++++++++++++++++++--------- defaults/main.yml | 39 ++++++++++++++++++- files/kill-rogue-jobs | 74 +++++++++++++++++++++++++++++++++++++ handlers/main.yml | 3 +- meta/main.yml | 54 +++++++++------------------ tasks/main.yml | 76 +++++++++++++++++++++++++++++++++++++- templates/rserver.conf.j2 | 3 ++ templates/rsession.conf.j2 | 4 ++ 8 files changed, 258 insertions(+), 56 deletions(-) create mode 100644 files/kill-rogue-jobs create mode 100644 templates/rserver.conf.j2 create mode 100644 templates/rsession.conf.j2 diff --git a/README.md b/README.md index 3637db8..2e82118 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,58 @@ Role Name ========= -A brief description of the role goes here. - -Requirements ------------- - -Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required. +A role that installs the free version of Rstudio Server Role Variables -------------- -A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well. +The most important variables are listed below: -Dependencies ------------- +``` yaml +rstudio_install_server: True +rstudio_enabled: '{{ rstudio_install_server }}' +rstudio_server_version: '1.4.1106' +rstudio_file: 'rstudio-server-{{ rstudio_server_version }}-amd64.deb' +rstudio_download_url: 'https://download2.rstudio.org/server/bionic/amd64/{{ rstudio_file }}' -A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles. +rstudio_install_kill_script: True +# cron job minutes +rstudio_kill_script_frequency: "*/5" -Example Playbook ----------------- +r_session_timeout_minutes: 360 +r_session_disconnected_timeout_minutes: 360 +r_session_quit_child_processes_on_exit: 0 +r_session_default_working_dir: '~' +r_session_default_new_project_dir: '~' +r_session_save_action_default: 'yes' +r_session_allow_shell: 1 +r_session_allow_terminal_websockets: 1 +r_session_limit_cpu_time_minutes: 0 +r_session_limit_file_upload_size_mb: 0 +r_session_limit_xfs_quota: 'no' -Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too: +rstudio_rserver_conf_opts: + - { name: 'r-cran-repos', value: 'http://cran.mirror.garr.it/mirrors/CRAN/' } + - { name: 'session-timeout-minutes', value: '{{ r_session_timeout_minutes }}' } + - { name: 'session-disconnected-timeout-minutes', value: '{{ r_session_disconnected_timeout_minutes }}' } + - { name: 'session-quit-child-processes-on-exit', value: '{{ r_session_quit_child_processes_on_exit }}' } + - { name: 'session-default-working-dir', value: '{{ r_session_default_working_dir }}' } + - { name: 'session-default-new-project-dir', value: '{{ r_session_default_new_project_dir }}' } + - { name: 'session-save-action-default', value: '{{ r_session_save_action_default}}' } + - { name: 'allow-shell', value: '{{ r_session_allow_shell }}' } + - { name: 'allow-terminal-websockets', value: '{{ r_session_allow_terminal_websockets }}' } + - { name: 'limit-cpu-time-minutes', value: '{{ r_session_limit_cpu_time_minutes }}' } + - { name: 'limit-file-upload-size-mb', value: '{{ r_session_limit_file_upload_size_mb }}' } + - { name: 'limit-xfs-disk-quota', value: '{{ r_session_limit_xfs_quota }}' } + +rstudio_rsession_conf_opts: + - { name: 'www-address', value: '0.0.0.0' } +``` + +Dependencies +------------ - - hosts: servers - roles: - - { role: username.rolename, x: 42 } +None License ------- @@ -35,4 +62,4 @@ EUPL-1.2 Author Information ------------------ -An optional section for the role authors to include contact information, or a website (HTML is not allowed). +Andrea Dell'Amico, diff --git a/defaults/main.yml b/defaults/main.yml index 95d3c70..4620412 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,2 +1,39 @@ --- -# defaults file for ansible-role-template \ No newline at end of file +rstudio_install_server: True +rstudio_enabled: '{{ rstudio_install_server }}' +rstudio_server_version: '1.4.1106' +rstudio_file: 'rstudio-server-{{ rstudio_server_version }}-amd64.deb' +rstudio_download_url: 'https://download2.rstudio.org/server/bionic/amd64/{{ rstudio_file }}' + +rstudio_install_kill_script: True +# cron job minutes +rstudio_kill_script_frequency: "*/5" + +r_session_timeout_minutes: 360 +r_session_disconnected_timeout_minutes: 360 +r_session_quit_child_processes_on_exit: 0 +r_session_default_working_dir: '~' +r_session_default_new_project_dir: '~' +r_session_save_action_default: 'yes' +r_session_allow_shell: 1 +r_session_allow_terminal_websockets: 1 +r_session_limit_cpu_time_minutes: 0 +r_session_limit_file_upload_size_mb: 0 +r_session_limit_xfs_quota: 'no' + +rstudio_rserver_conf_opts: + - { name: 'r-cran-repos', value: 'http://cran.mirror.garr.it/mirrors/CRAN/' } + - { name: 'session-timeout-minutes', value: '{{ r_session_timeout_minutes }}' } + - { name: 'session-disconnected-timeout-minutes', value: '{{ r_session_disconnected_timeout_minutes }}' } + - { name: 'session-quit-child-processes-on-exit', value: '{{ r_session_quit_child_processes_on_exit }}' } + - { name: 'session-default-working-dir', value: '{{ r_session_default_working_dir }}' } + - { name: 'session-default-new-project-dir', value: '{{ r_session_default_new_project_dir }}' } + - { name: 'session-save-action-default', value: '{{ r_session_save_action_default}}' } + - { name: 'allow-shell', value: '{{ r_session_allow_shell }}' } + - { name: 'allow-terminal-websockets', value: '{{ r_session_allow_terminal_websockets }}' } + - { name: 'limit-cpu-time-minutes', value: '{{ r_session_limit_cpu_time_minutes }}' } + - { name: 'limit-file-upload-size-mb', value: '{{ r_session_limit_file_upload_size_mb }}' } + - { name: 'limit-xfs-disk-quota', value: '{{ r_session_limit_xfs_quota }}' } + +rstudio_rsession_conf_opts: + - { name: 'www-address', value: '0.0.0.0' } diff --git a/files/kill-rogue-jobs b/files/kill-rogue-jobs new file mode 100644 index 0000000..c6df0b4 --- /dev/null +++ b/files/kill-rogue-jobs @@ -0,0 +1,74 @@ +#!/bin/bash +# +# TODO: +# - print nagios friendly output into a file +# - kill rsession processes older than N days +# + +set -e +set -o pipefail + +CMD_NAME=$0 + +USER_N= +USER_PROCS_TO_KILL= +SPARE_MEM=1048576 +USERS_SESSIONS=$( ps -edaf | grep rsession | grep -v defunct | grep -v grep | awk '{ print $10 }' | uniq ) +if [ -z "$USERS_SESSIONS" ] ; then + eval logger '$CMD_NAME: There are no active sessions' + exit 0 +fi + +OUT_DIR=$( mktemp -d -t kill-rogue-jobs.XXXXXXXXXX ) +USER_PROCS_LIST=$OUT_DIR/proclist +USER_PROCS_PARENTS=$OUT_DIR/parents + +trap "eval logger '$CMD_NAME: trap intercepted, exiting.' ; cleanup" SIGHUP SIGINT SIGTERM + +function cleanup() { + rm -fr $OUT_DIR +} + +function find_rogue_processes() { + eval logger '$CMD_NAME: find_rogue_processes for user $USER_N' + ps -edaf | grep rsession | grep -v grep | grep ${USER_N} | awk '{ print $3 }' | uniq > $USER_PROCS_PARENTS + ps -edaf | grep rsession | grep -v grep | grep ${USER_N} | awk '{ print $2 }' | uniq > $USER_PROCS_LIST + for parent in $( cat $USER_PROCS_PARENTS ) ; do + grep -v $parent $USER_PROCS_LIST > $USER_PROCS_LIST.tmp + mv $USER_PROCS_LIST.tmp $USER_PROCS_LIST + done + USER_PROCS_TO_KILL=$( cat $USER_PROCS_LIST ) +} + +function exterminate() { + eval logger '$CMD_NAME: exterminate killing user $USER_N processes' + for pid in $( echo $USER_PROCS_TO_KILL ) ; do + kill -15 $pid + done +} + +NUM_CPUS=$( grep processor /proc/cpuinfo | wc -l ) +ALLOWED_THREADS=$(( $NUM_CPUS - 1 )) +TOTAL_MEM=$( grep MemTotal /proc/meminfo | awk '{ print $2 }' ) +ALLOWED_USED_MEM=$(( $TOTAL_MEM - $SPARE_MEM )) + +for USER_N in $( echo $USERS_SESSIONS ) ; do + USER_PROCS=$( ps -edaf | grep rsession | grep -v grep | grep ${USER_N} | wc -l ) + USER_MEM=$( ps -eo pid,rss,vsz,args | grep rsession | grep -v grep | grep ${USER_N} | awk '{ print $2}' | paste -sd+ | bc ) + if [ $USER_PROCS -gt $ALLOWED_THREADS ] || [ $USER_MEM -gt $ALLOWED_USED_MEM ] ; then + if [ $USER_PROCS -gt $ALLOWED_THREADS ] ; then + eval logger '$CMD_NAME: user $USER_N is running too many processes' + fi + if [ $USER_MEM -gt $ALLOWED_USED_MEM ] ; then + eval logger '$CMD_NAME: user $USER_N is using too much memory' + fi + find_rogue_processes + exterminate + else + eval logger '$CMD_NAME: we do not need to kill any processes for user $USER_N' + fi +done + +trap cleanup EXIT +exit 0 + diff --git a/handlers/main.yml b/handlers/main.yml index 27474e0..e80a592 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -1,2 +1,3 @@ --- -# handlers file for ansible-role-template \ No newline at end of file +- name: Restart Rstudio Server + service: name=rstudio-server state=restarted \ No newline at end of file diff --git a/meta/main.yml b/meta/main.yml index 81bda14..6dfa39d 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,46 +1,28 @@ galaxy_info: - author: your name - description: your description - company: your company (optional) + author: Andrea Dell'Amico + description: Systems Architect + company: ISTI-CNR - # If the issue tracker for your role is not on github, uncomment the - # next line and provide a value - issue_tracker_url: https://support.d4science.org/projects/automatic-provisioning/issues + issue_tracker_url: https://redmine-s2i2s.isti.cnr.it/projects/provisioning - license: EUPL-1.2 + license: EUPL 1.2+ min_ansible_version: 2.8 - # If this a Container Enabled role, provide the minimum Ansible Container version. - # min_ansible_container_version: - - # Optionally specify the branch Galaxy will use when accessing the GitHub - # repo for this role. During role install, if no tags are available, - # Galaxy will use this branch. During import Galaxy will access files on - # this branch. If Travis integration is configured, only notifications for this - # branch will be accepted. Otherwise, in all cases, the repo's default branch - # (usually master) will be used. - #github_branch: - - # - # Provide a list of supported platforms, and for each platform a list of versions. - # If you don't wish to enumerate all versions for a particular platform, use 'all'. # To view available platforms and versions (or releases), visit: # https://galaxy.ansible.com/api/v1/platforms/ # platforms: - - name: Ubuntu - versions: - - bionic - - galaxy_tags: [] - # List tags for your role here, one per line. A tag is a keyword that describes - # and categorizes the role. Users find roles by searching for tags. Be sure to - # remove the '[]' above, if you add tags to this list. - # - # NOTE: A tag is limited to a single word comprised of alphanumeric characters. - # Maximum 20 tags per role. - -dependencies: [] - # List your role dependencies here, one per line. Be sure to remove the '[]' above, - # if you add dependencies to this list. \ No newline at end of file + - name: Ubuntu + versions: + - bionic + + galaxy_tags: + - R + - rstudio + +dependencies: + - src: git+https://code-repo.d4science.org/InfraScience/ansible-role-R.git + version: master + name: R + state: latest diff --git a/tasks/main.yml b/tasks/main.yml index 53c6cae..a5093b4 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,2 +1,76 @@ --- -# tasks file for ansible-role-template \ No newline at end of file +- block: + - name: Install the gdebi tool + apt: pkg=gdebi-core state=latest update_cache=yes cache_valid_time=1800 + + - name: Install the apparmor utilities + apt: pkg=apparmor-utils state=present update_cache=yes cache_valid_time=1800 + + - name: Download the rstudio server deb package +# get_url: url={{ rstudio_download_url }} dest=/srv/{{ rstudio_file }} + shell: cd /srv ; wget {{ rstudio_download_url }} + args: + creates: /srv/{{ rstudio_file }} + register: rstudio_download + + - name: Install the rstudio server package + command: gdebi -n -q /srv/{{ rstudio_file }} + when: rstudio_download is changed + + - name: Change the apparmor behaviour to complain, otherwise it cannot read the nslcd socket + shell: RETVAL='' ; if [ -x /usr/sbin/apparmor_status ] ; then APPARMOR_STATE=$( /usr/sbin/apparmor_status ) ; RETVAL=$? ; if [ $RETVAL -eq 0 ] ; then aa-complain /usr/lib/rstudio-server/bin/rserver ; fi ; fi + when: rstudio_enabled + + - name: Install bc, needed by the script that kills abusive jobs + apt: pkg=bc state=present update_cache=yes cache_valid_time=3600 + when: rstudio_install_kill_script + tags: [ 'rstudio', 'rstudio_server', 'rstudio_kill_rogues' ] + + - name: Install a script that kills the abusive job processes + copy: src=kill-rogue-jobs dest=/usr/local/bin/kill-rogue-jobs owner=root group=root mode=0755 + when: rstudio_install_kill_script + tags: [ 'rstudio', 'rstudio_server', 'rstudio_kill_rogues' ] + + - name: Install a cron job that kills the abusive jobs + cron: name="Kill rogue jobs" job="/usr/local/bin/kill-rogue-jobs" user=root minute="{{ rstudio_kill_script_frequency }}" + when: rstudio_install_kill_script + tags: [ 'rstudio', 'rstudio_server', 'rstudio_kill_rogues' ] + + - name: Install the rstudio server configuration + template: src={{ item }}.j2 dest=/etc/rstudio/{{ item }} owner=root group=root mode=0644 + with_items: + - rsession.conf + - rserver.conf + notify: Restart Rstudio Server + tags: [ 'rstudio', 'rstudio_server', 'rstudio_server_conf' ] + + - name: Ensure that rstudio server is enabled and running + service: name=rstudio-server state=started enabled=yes + when: rstudio_enabled + + - name: Ensure that rstudio server is disabled and stopped + service: name=rstudio-server state=stopped enabled=no + when: not rstudio_enabled + + when: rstudio_install_server + tags: [ 'rstudio', 'rstudio_server' ] + + +- block: + - name: Ensure that rstudio server is disabled and stopped + service: name=rstudio-server state=stopped enabled=no + + - name: Remove the rstudio-server and gdebi packages + apt: pkg={{ item }} state=absent + with_items: + - gdebi + - rstudio-server + + - name: Remove the script that kills the abusive job processes + file: dest=/usr/local/bin/kill-rogue-jobs state=absent + + - name: Remove the cron job that kills the abusive jobs + cron: name="Kill rogue jobs" job="/usr/local/bin/kill-rogue-jobs" user=root minute="{{ rstudio_kill_script_frequency }}" state=absent + + when: not rstudio_install_server + tags: [ 'rstudio', 'rstudio_server' ] diff --git a/templates/rserver.conf.j2 b/templates/rserver.conf.j2 new file mode 100644 index 0000000..c231f67 --- /dev/null +++ b/templates/rserver.conf.j2 @@ -0,0 +1,3 @@ +{% for rstudio_opt in rstudio_rserver_conf_opts %} +{{ rstudio_opt.name }}={{ rstudio_opt.value }} +{% endfor %} diff --git a/templates/rsession.conf.j2 b/templates/rsession.conf.j2 new file mode 100644 index 0000000..17647f4 --- /dev/null +++ b/templates/rsession.conf.j2 @@ -0,0 +1,4 @@ +# R Session Configuration File +{% for rsession_opt in rstudio_rsession_conf_opts %} +{{ rsession_opt.name }}={{ rsession_opt.value }} +{% endfor %}