diff --git a/install-r-packages.cron b/install-r-packages.cron new file mode 100644 index 0000000..f2a879e --- /dev/null +++ b/install-r-packages.cron @@ -0,0 +1 @@ +*/10 5-23,1-2 * * * root /usr/local/bin/update_r_packages install >/var/log/install_r_packages 2>&1 diff --git a/update-rprofile.cron b/update-rprofile.cron new file mode 100644 index 0000000..dc4c00c --- /dev/null +++ b/update-rprofile.cron @@ -0,0 +1 @@ +* 3 * * * root cd /srv/d4science/RStudioConfiguration ; svn update>/var/log/update_rprofile 2>&1 \ No newline at end of file diff --git a/update_r_packages b/update_r_packages new file mode 100644 index 0000000..ec01c0a --- /dev/null +++ b/update_r_packages @@ -0,0 +1,239 @@ +#!/bin/bash +# +# TODO: kill an old process if it is running from too much time (12 hours?) +# using something like ps -o etimes= -p "$PROCNUM" +# + +export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin" + +RETVAl= +PARAMS=$# +ACTION=$1 +PROCNUM=$$ +OLDPROC= +OLDPROC_RUNNING= +LOCKDIR=/var/run +LOCK_FILE=$LOCKDIR/.update_r_pkgs.lock +TMP_FILES_DIR=/var/tmp/r_pkgs_update +# We cannot answer questions +DEBIAN_FRONTEND=noninteractive +R_CRAN_MIRROR=https://cran.mirror.garr.it/mirrors/CRAN/ +R_PKGS_FROM_SVN=True +R_PKGS_SVN_DIR=RPackagesManagement +R_PKGS_SVN_URL=http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/ +R_PKGS_SVN_BASE_DIR=/srv/r_updater +R_PKGS_FILES_PREFIX= +SVN_UPDATE_STATUS= +# In seconds. 60*60*6=21600s (6h) +UPDATER_PROCESS_MAX_RUNTIME=21600 +# - debian packages list format: +# one package per line +DEB_PKGS_SKIP=0 +DEBIAN_PKGS_LIST_URL= +PKGS_LIST= +# - R packages list format: +# name[:mirror] +CRAN_PKGS_SKIP=0 +R_PKGS_LIST_URL= +R_PKGS_LIST= +# - R packages from github list format: +# - owner/package +GITHUB_PKGS_SKIP=0 +R_PKGS_FROM_GITHUB_LIST_URL= +R_PKGS_GITHUB= + +trap "logger 'update_r_packages: trap intercepted, exiting.' ; cleanup" SIGHUP SIGINT SIGTERM + +function cleanup() { + logger "update_r_packages: cleaning up" + rm -f $LOCK_FILE + rm -fr $TMP_FILES_DIR +} + +function usage() { + if [ $PARAMS -ne 1 ] ; then + echo "Need at least an argument: 'upgrade' or 'install'." + echo "- 'upgrade' installs new packages and upgrades the existin ones when needed." + echo "- 'install' installs new packages." + cleanup + exit 1 + fi +} + +function get_args() { + if [ "$ACTION" != "upgrade" -a "$ACTION" != "install" ] ; then + usage + fi +} + +function fail() { + logger "update_r_packages: Something went wrong, exiting." + cleanup + exit 1 +} + +function init_env() { + if [ -f $LOCK_FILE ] ; then + OLDPROC=$( cat $LOCK_FILE ) + OLDPROC_RUNNING=$( ps auwwx | grep -v grep | grep $OLDPROC | awk '{ print $2 }' ) + RETVAL=$? + if [ ! -z "$OLDPROC_RUNNING" ] ; then + logger "update_r_packages: pid of the already running process: $OLDPROC_RUNNING" + OLDPROC_RUNNING_TIME=$( ps -o etimes= -p ${OLDPROC_RUNNING} ) + if [ $OLDPROC_RUNNING_TIME -gt $UPDATER_PROCESS_MAX_RUNTIME ] ; then + logger "update_r_packages: process $OLDPROC_RUNNING was running for $OLDPROC_RUNNING_TIME seconds. Got stuck, killing it" + kill -9 $OLDPROC_RUNNING + cleanup + else + logger "update_r_packages: another process is running, exiting." + exit 0 + fi + else + logger "update_r_packages: lock file exist but the process not. Continuing." + rm -fr $TMP_FILES_DIR + fi + else + logger 'update_r_packages: no other jobs running, proceeding.' + fi + RETVAL= + echo "$PROCNUM" > $LOCK_FILE + mkdir -p $TMP_FILES_DIR +} + +function get_data_files() { + logger "update_r_packages: get the single files from http." + # Get the packages list + if [ -z $DEBIAN_PKGS_LIST_URL ] ; then + DEB_PKGS_SKIP=1 + logger "update_r_packages: the debian packages list is not available." + else + PKGS_LIST=$( mktemp $TMP_FILES_DIR/rdebs.XXXXXXX ) + logger "update_r_packages: getting the debian packages list." + wget -q -o /dev/null -O $PKGS_LIST $DEBIAN_PKGS_LIST_URL + fi + if [ -z $R_PKGS_LIST_URL ] ; then + CRAN_PKGS_SKIP=1 + logger "update_r_packages: the CRAN packages list is not available." + else + R_PKGS_LIST=$( mktemp $TMP_FILES_DIR/rpkgs.XXXXXXX ) + logger "update_r_packages: getting the R packages list that will be installed from CRAN" + wget -q -o /dev/null -O $R_PKGS_LIST $R_PKGS_LIST_URL + fi + if [ -z $R_PKGS_FROM_GITHUB_LIST_URL ] ; then + GITHUB_PKGS_SKIP=1 + logger "update_r_packages: the Github packages list is not available." + else + R_PKGS_GITHUB=$( mktemp $TMP_FILES_DIR/rpkgsgithub.XXXXXXX ) + logger "update_r_packages: getting the R packages list that will be installed from github" + wget -q -o /dev/null -O $R_PKGS_GITHUB $R_PKGS_FROM_GITHUB_LIST_URL + fi +} + +function get_data_files_from_svn() { + logger "update_r_packages: files from a SVN repo." + if [ -d $R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR ] ; then + logger "update_r_packages: SVN update" + cd $R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR + SVN_CLEANUP_OP=$( svn cleanup ) + SVN_UPDATE_OP=$( svn update | tail -1 | grep Updated >/dev/null 2>&1 ) + SVN_UPDATE_STATUS=$? + else + cd $R_PKGS_SVN_BASE_DIR + logger "update_r_packages: first SVN checkout." + svn co $R_PKGS_SVN_URL >/dev/null 2>&1 + fi + PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_deb_pkgs.txt + R_PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_cran_pkgs.txt + R_PKGS_GITHUB=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_github_pkgs.txt +} + +function debian_pkgs() { + if [ $DEB_PKGS_SKIP -eq 0 ] ; then + # Update the apt cache and install the packages in non interactive mode + logger "update_r_packages: Installing the debian dependencies" + if [ -z "$(find /var/cache/apt/pkgcache.bin -mmin -360)" ]; then + apt-get update -q >/dev/null 2>&1 + else + logger "update_r_packages: APT cache not updated" + fi + >/var/log/update_r_debs.log + while read deb_pkg ; do + apt-get install ${deb_pkg} -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" >>/var/log/update_r_debs.log 2>&1 + done < $PKGS_LIST + apt-get autoremove -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" >> /var/log/update_r_debs.log 2>&1 + else + logger "update_r_packages: skipping the debian packages installation" + fi +} + +function remove_r_install_packages_lock_files() { + # install.packages leaves lock files around if the process crashes + rm -fr /usr/local/lib/R/site-library/00LOCK-* + } + +function r_cran_pkgs() { + if [ $CRAN_PKGS_SKIP -eq 0 ] ; then + logger "update_r_packages: Installing R packages from CRAN" + for l in $( cat $R_PKGS_LIST ) ; do + pkg=$( echo $l | cut -d : -f 1 ) + is_mirror_ret= + is_mirror=$( echo $l | grep ':' ) + is_mirror_ret=$? + if [ $is_mirror_ret -eq 0 ] ; then + mirror=$( echo $l | cut -d : -f 2- ) + else + mirror=$R_CRAN_MIRROR + fi + if [ "$ACTION" == "upgrade" ] ; then + Rscript --slave --no-save --no-restore-history -e "install.packages(pkgs='$pkg', repos=c('$mirror/'));" + else + Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { install.packages(pkgs='$pkg', repos=c('$mirror/')); }" + fi + done + else + logger "update_r_packages: skipping the R CRAN packages installation" + fi +} + +function r_github_pkgs() { + if [ $GITHUB_PKGS_SKIP -eq 0 ] ; then + logger "update_r_packages: Installing R packages from Github" + for l in $( cat $R_PKGS_GITHUB ) ; do + pkg=$( echo $l | cut -d "/" -f 2 ) + if [ "$ACTION" == "upgrade" ] ; then + #Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('$l');" + Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); require(jsonlite) ; package_to_install <- '$l' ; refs <- jsonlite::read_json(sprintf('https://api.github.com/repos/%s/releases', package_to_install)) ; ref_to_install <- 'master'; if(length(refs)>0) { ref_to_install <- refs[[1]][['tag_name']] } ; devtools::install_github(package_to_install, ref = ref_to_install)" + + else + #Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { require(devtools); require(methods) ; install_github('$l'); }" + Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { require(devtools); require(methods); require(jsonlite) ; package_to_install <- '$l' ; refs <- jsonlite::read_json(sprintf('https://api.github.com/repos/%s/releases', package_to_install)) ; ref_to_install <- 'master'; if(length(refs)>0) { ref_to_install <- refs[[1]][['tag_name']] } ; devtools::install_github(package_to_install, ref = ref_to_install) }" + fi + done + else + logger "update_r_packages: skipping the R GitHub packages installation" + fi +} + +######### +# Main +# + +usage +get_args +init_env +if [ $R_PKGS_FROM_SVN == 'True' ] ; then + get_data_files_from_svn + if [ $SVN_UPDATE_STATUS -ne 0 -a "$ACTION" == "install" ] ; then + logger "update_r_packages: nothing new to install from SVN, exiting" + cleanup + exit 0 + fi +else + get_data_files +fi +debian_pkgs +remove_r_install_packages_lock_files +r_cran_pkgs +r_github_pkgs +cleanup +exit 0 diff --git a/upgrade-r-packages.cron b/upgrade-r-packages.cron new file mode 100644 index 0000000..c349c17 --- /dev/null +++ b/upgrade-r-packages.cron @@ -0,0 +1 @@ +7 3 * * * root /usr/local/bin/update_r_packages upgrade >/var/log/update_r_packages 2>&1