#!/bin/bash # # TODO: kill an old process if it is running from too much time (12 hours?) # using something like ps -o etimes= -p "$PROCNUM" # export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" RETVAl= PARAMS=$# ACTION=$1 PROCNUM=$$ OLDPROC= OLDPROC_RUNNING= LOCKDIR=/var/run LOCK_FILE=$LOCKDIR/.update_r_pkgs.lock TMP_FILES_DIR=/var/tmp/r_pkgs_update # We cannot answer questions DEBIAN_FRONTEND=noninteractive R_CRAN_MIRROR=https://cran.mirror.garr.it/mirrors/CRAN/ R_PKGS_FROM_SVN=True R_PKGS_SVN_DIR=RPackagesManagement R_PKGS_SVN_URL=http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ R_PKGS_SVN_BASE_DIR=/srv/r_updater R_PKGS_FILES_PREFIX= SVN_UPDATE_STATUS= # In seconds. 60*60*6=21600s (6h) UPDATER_PROCESS_MAX_RUNTIME=21600 # - debian packages list format: # one package per line DEB_PKGS_SKIP=0 DEBIAN_PKGS_LIST_URL= PKGS_LIST= # - R packages list format: # name[:mirror] CRAN_PKGS_SKIP=0 R_PKGS_LIST_URL= R_PKGS_LIST= # - R packages from github list format: # - owner/package GITHUB_PKGS_SKIP=0 R_PKGS_FROM_GITHUB_LIST_URL= R_PKGS_GITHUB= trap "logger 'update_r_packages: trap intercepted, exiting.' ; cleanup" SIGHUP SIGINT SIGTERM function cleanup() { logger "update_r_packages: cleaning up" rm -f $LOCK_FILE rm -fr $TMP_FILES_DIR } function usage() { if [ $PARAMS -ne 1 ] ; then echo "Need at least an argument: 'upgrade' or 'install'." echo "- 'upgrade' installs new packages and upgrades the existin ones when needed." echo "- 'install' installs new packages." cleanup exit 1 fi } function get_args() { if [ "$ACTION" != "upgrade" -a "$ACTION" != "install" ] ; then usage fi } function fail() { logger "update_r_packages: Something went wrong, exiting." cleanup exit 1 } function init_env() { if [ -f $LOCK_FILE ] ; then OLDPROC=$( cat $LOCK_FILE ) OLDPROC_RUNNING=$( ps auwwx | grep -v grep | grep $OLDPROC | awk '{ print $2 }' ) RETVAL=$? if [ ! -z "$OLDPROC_RUNNING" ] ; then logger "update_r_packages: pid of the already running process: $OLDPROC_RUNNING" OLDPROC_RUNNING_TIME=$( ps -o etimes= -p ${OLDPROC_RUNNING} ) if [ $OLDPROC_RUNNING_TIME -gt $UPDATER_PROCESS_MAX_RUNTIME ] ; then logger "update_r_packages: process $OLDPROC_RUNNING was running for $OLDPROC_RUNNING_TIME seconds. Got stuck, killing it" kill -9 $OLDPROC_RUNNING cleanup else logger "update_r_packages: another process is running, exiting." exit 0 fi else logger "update_r_packages: lock file exist but the process not. Continuing." rm -fr $TMP_FILES_DIR fi else logger 'update_r_packages: no other jobs running, proceeding.' fi RETVAL= echo "$PROCNUM" > $LOCK_FILE mkdir -p $TMP_FILES_DIR } function get_data_files() { logger "update_r_packages: get the single files from http." # Get the packages list if [ -z $DEBIAN_PKGS_LIST_URL ] ; then DEB_PKGS_SKIP=1 logger "update_r_packages: the debian packages list is not available." else PKGS_LIST=$( mktemp $TMP_FILES_DIR/rdebs.XXXXXXX ) logger "update_r_packages: getting the debian packages list." wget -q -o /dev/null -O $PKGS_LIST $DEBIAN_PKGS_LIST_URL fi if [ -z $R_PKGS_LIST_URL ] ; then CRAN_PKGS_SKIP=1 logger "update_r_packages: the CRAN packages list is not available." else R_PKGS_LIST=$( mktemp $TMP_FILES_DIR/rpkgs.XXXXXXX ) logger "update_r_packages: getting the R packages list that will be installed from CRAN" wget -q -o /dev/null -O $R_PKGS_LIST $R_PKGS_LIST_URL fi if [ -z $R_PKGS_FROM_GITHUB_LIST_URL ] ; then GITHUB_PKGS_SKIP=1 logger "update_r_packages: the Github packages list is not available." else R_PKGS_GITHUB=$( mktemp $TMP_FILES_DIR/rpkgsgithub.XXXXXXX ) logger "update_r_packages: getting the R packages list that will be installed from github" wget -q -o /dev/null -O $R_PKGS_GITHUB $R_PKGS_FROM_GITHUB_LIST_URL fi } function get_data_files_from_svn() { logger "update_r_packages: files from a SVN repo." if [ -d $R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR ] ; then logger "update_r_packages: SVN update" cd $R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR SVN_CLEANUP_OP=$( svn cleanup ) SVN_UPDATE_OP=$( svn update | tail -1 | grep Updated >/dev/null 2>&1 ) SVN_UPDATE_STATUS=$? else cd $R_PKGS_SVN_BASE_DIR logger "update_r_packages: first SVN checkout." svn co $R_PKGS_SVN_URL >/dev/null 2>&1 fi PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_deb_pkgs.txt R_PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_cran_pkgs.txt R_PKGS_GITHUB=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_github_pkgs.txt } function debian_pkgs() { if [ $DEB_PKGS_SKIP -eq 0 ] ; then # Update the apt cache and install the packages in non interactive mode logger "update_r_packages: Installing the debian dependencies" if [ -z "$(find /var/cache/apt/pkgcache.bin -mmin -360)" ]; then apt-get update -q >/dev/null 2>&1 else logger "update_r_packages: APT cache not updated" fi >/var/log/update_r_debs.log while read deb_pkg ; do apt-get install ${deb_pkg} -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" >>/var/log/update_r_debs.log 2>&1 done < $PKGS_LIST apt-get autoremove -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" >> /var/log/update_r_debs.log 2>&1 else logger "update_r_packages: skipping the debian packages installation" fi } function remove_r_install_packages_lock_files() { # install.packages leaves lock files around if the process crashes rm -fr /usr/local/lib/R/site-library/00LOCK-* } function r_cran_pkgs() { if [ $CRAN_PKGS_SKIP -eq 0 ] ; then logger "update_r_packages: Installing R packages from CRAN" for l in $( cat $R_PKGS_LIST ) ; do pkg=$( echo $l | cut -d : -f 1 ) is_mirror_ret= is_mirror=$( echo $l | grep ':' ) is_mirror_ret=$? if [ $is_mirror_ret -eq 0 ] ; then mirror=$( echo $l | cut -d : -f 2- ) else mirror=$R_CRAN_MIRROR fi if [ "$ACTION" == "upgrade" ] ; then Rscript --slave --no-save --no-restore-history -e "install.packages(pkgs='$pkg', repos=c('$mirror/'));" else Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { install.packages(pkgs='$pkg', repos=c('$mirror/')); }" fi done else logger "update_r_packages: skipping the R CRAN packages installation" fi } function r_github_pkgs() { if [ $GITHUB_PKGS_SKIP -eq 0 ] ; then logger "update_r_packages: Installing R packages from Github" for l in $( cat $R_PKGS_GITHUB ) ; do pkg=$( echo $l | cut -d "/" -f 2 ) if [ "$ACTION" == "upgrade" ] ; then #Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('$l');" Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); require(jsonlite) ; package_to_install <- '$l' ; refs <- jsonlite::read_json(sprintf('https://api.github.com/repos/%s/releases', package_to_install)) ; ref_to_install <- 'master'; if(length(refs)>0) { ref_to_install <- refs[[1]][['tag_name']] } ; devtools::install_github(package_to_install, ref = ref_to_install)" else #Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { require(devtools); require(methods) ; install_github('$l'); }" Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { require(devtools); require(methods); require(jsonlite) ; package_to_install <- '$l' ; refs <- jsonlite::read_json(sprintf('https://api.github.com/repos/%s/releases', package_to_install)) ; ref_to_install <- 'master'; if(length(refs)>0) { ref_to_install <- refs[[1]][['tag_name']] } ; devtools::install_github(package_to_install, ref = ref_to_install) }" fi done else logger "update_r_packages: skipping the R GitHub packages installation" fi } ######### # Main # usage get_args init_env if [ $R_PKGS_FROM_SVN == 'True' ] ; then get_data_files_from_svn if [ $SVN_UPDATE_STATUS -ne 0 -a "$ACTION" == "install" ] ; then logger "update_r_packages: nothing new to install from SVN, exiting" cleanup exit 0 fi else get_data_files fi debian_pkgs remove_r_install_packages_lock_files r_cran_pkgs r_github_pkgs cleanup exit 0