Add other files. Change the Dockerfile to install the cron jobs that install/upgrade the R packages and that upgrade the main Rprofile.
This commit is contained in:
Andrea Dell'Amico
parent
6f9c6d4487
commit
fca12140c0
|
|
@ -0,0 +1 @@
|
||||||
|
*/10 5-23,1-2 * * * root /usr/local/bin/update_r_packages install >/var/log/install_r_packages 2>&1
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
* 3 * * * root cd /srv/d4science/RStudioConfiguration ; svn update>/var/log/update_rprofile 2>&1
|
||||||
|
|
@ -0,0 +1,239 @@
|
||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# TODO: kill an old process if it is running from too much time (12 hours?)
|
||||||
|
# using something like ps -o etimes= -p "$PROCNUM"
|
||||||
|
#
|
||||||
|
|
||||||
|
export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
|
||||||
|
|
||||||
|
RETVAl=
|
||||||
|
PARAMS=$#
|
||||||
|
ACTION=$1
|
||||||
|
PROCNUM=$$
|
||||||
|
OLDPROC=
|
||||||
|
OLDPROC_RUNNING=
|
||||||
|
LOCKDIR=/var/run
|
||||||
|
LOCK_FILE=$LOCKDIR/.update_r_pkgs.lock
|
||||||
|
TMP_FILES_DIR=/var/tmp/r_pkgs_update
|
||||||
|
# We cannot answer questions
|
||||||
|
DEBIAN_FRONTEND=noninteractive
|
||||||
|
R_CRAN_MIRROR=https://cran.mirror.garr.it/mirrors/CRAN/
|
||||||
|
R_PKGS_FROM_SVN=True
|
||||||
|
R_PKGS_SVN_DIR=RPackagesManagement
|
||||||
|
R_PKGS_SVN_URL=http://svn.research-infrastructures.eu/public/d4science/gcube/trunk/data-analysis/RConfiguration/RPackagesManagement/
|
||||||
|
R_PKGS_SVN_BASE_DIR=/srv/r_updater
|
||||||
|
R_PKGS_FILES_PREFIX=
|
||||||
|
SVN_UPDATE_STATUS=
|
||||||
|
# In seconds. 60*60*6=21600s (6h)
|
||||||
|
UPDATER_PROCESS_MAX_RUNTIME=21600
|
||||||
|
# - debian packages list format:
|
||||||
|
# one package per line
|
||||||
|
DEB_PKGS_SKIP=0
|
||||||
|
DEBIAN_PKGS_LIST_URL=
|
||||||
|
PKGS_LIST=
|
||||||
|
# - R packages list format:
|
||||||
|
# name[:mirror]
|
||||||
|
CRAN_PKGS_SKIP=0
|
||||||
|
R_PKGS_LIST_URL=
|
||||||
|
R_PKGS_LIST=
|
||||||
|
# - R packages from github list format:
|
||||||
|
# - owner/package
|
||||||
|
GITHUB_PKGS_SKIP=0
|
||||||
|
R_PKGS_FROM_GITHUB_LIST_URL=
|
||||||
|
R_PKGS_GITHUB=
|
||||||
|
|
||||||
|
trap "logger 'update_r_packages: trap intercepted, exiting.' ; cleanup" SIGHUP SIGINT SIGTERM
|
||||||
|
|
||||||
|
function cleanup() {
|
||||||
|
logger "update_r_packages: cleaning up"
|
||||||
|
rm -f $LOCK_FILE
|
||||||
|
rm -fr $TMP_FILES_DIR
|
||||||
|
}
|
||||||
|
|
||||||
|
function usage() {
|
||||||
|
if [ $PARAMS -ne 1 ] ; then
|
||||||
|
echo "Need at least an argument: 'upgrade' or 'install'."
|
||||||
|
echo "- 'upgrade' installs new packages and upgrades the existin ones when needed."
|
||||||
|
echo "- 'install' installs new packages."
|
||||||
|
cleanup
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_args() {
|
||||||
|
if [ "$ACTION" != "upgrade" -a "$ACTION" != "install" ] ; then
|
||||||
|
usage
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function fail() {
|
||||||
|
logger "update_r_packages: Something went wrong, exiting."
|
||||||
|
cleanup
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
function init_env() {
|
||||||
|
if [ -f $LOCK_FILE ] ; then
|
||||||
|
OLDPROC=$( cat $LOCK_FILE )
|
||||||
|
OLDPROC_RUNNING=$( ps auwwx | grep -v grep | grep $OLDPROC | awk '{ print $2 }' )
|
||||||
|
RETVAL=$?
|
||||||
|
if [ ! -z "$OLDPROC_RUNNING" ] ; then
|
||||||
|
logger "update_r_packages: pid of the already running process: $OLDPROC_RUNNING"
|
||||||
|
OLDPROC_RUNNING_TIME=$( ps -o etimes= -p ${OLDPROC_RUNNING} )
|
||||||
|
if [ $OLDPROC_RUNNING_TIME -gt $UPDATER_PROCESS_MAX_RUNTIME ] ; then
|
||||||
|
logger "update_r_packages: process $OLDPROC_RUNNING was running for $OLDPROC_RUNNING_TIME seconds. Got stuck, killing it"
|
||||||
|
kill -9 $OLDPROC_RUNNING
|
||||||
|
cleanup
|
||||||
|
else
|
||||||
|
logger "update_r_packages: another process is running, exiting."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
logger "update_r_packages: lock file exist but the process not. Continuing."
|
||||||
|
rm -fr $TMP_FILES_DIR
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
logger 'update_r_packages: no other jobs running, proceeding.'
|
||||||
|
fi
|
||||||
|
RETVAL=
|
||||||
|
echo "$PROCNUM" > $LOCK_FILE
|
||||||
|
mkdir -p $TMP_FILES_DIR
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_data_files() {
|
||||||
|
logger "update_r_packages: get the single files from http."
|
||||||
|
# Get the packages list
|
||||||
|
if [ -z $DEBIAN_PKGS_LIST_URL ] ; then
|
||||||
|
DEB_PKGS_SKIP=1
|
||||||
|
logger "update_r_packages: the debian packages list is not available."
|
||||||
|
else
|
||||||
|
PKGS_LIST=$( mktemp $TMP_FILES_DIR/rdebs.XXXXXXX )
|
||||||
|
logger "update_r_packages: getting the debian packages list."
|
||||||
|
wget -q -o /dev/null -O $PKGS_LIST $DEBIAN_PKGS_LIST_URL
|
||||||
|
fi
|
||||||
|
if [ -z $R_PKGS_LIST_URL ] ; then
|
||||||
|
CRAN_PKGS_SKIP=1
|
||||||
|
logger "update_r_packages: the CRAN packages list is not available."
|
||||||
|
else
|
||||||
|
R_PKGS_LIST=$( mktemp $TMP_FILES_DIR/rpkgs.XXXXXXX )
|
||||||
|
logger "update_r_packages: getting the R packages list that will be installed from CRAN"
|
||||||
|
wget -q -o /dev/null -O $R_PKGS_LIST $R_PKGS_LIST_URL
|
||||||
|
fi
|
||||||
|
if [ -z $R_PKGS_FROM_GITHUB_LIST_URL ] ; then
|
||||||
|
GITHUB_PKGS_SKIP=1
|
||||||
|
logger "update_r_packages: the Github packages list is not available."
|
||||||
|
else
|
||||||
|
R_PKGS_GITHUB=$( mktemp $TMP_FILES_DIR/rpkgsgithub.XXXXXXX )
|
||||||
|
logger "update_r_packages: getting the R packages list that will be installed from github"
|
||||||
|
wget -q -o /dev/null -O $R_PKGS_GITHUB $R_PKGS_FROM_GITHUB_LIST_URL
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function get_data_files_from_svn() {
|
||||||
|
logger "update_r_packages: files from a SVN repo."
|
||||||
|
if [ -d $R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR ] ; then
|
||||||
|
logger "update_r_packages: SVN update"
|
||||||
|
cd $R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR
|
||||||
|
SVN_CLEANUP_OP=$( svn cleanup )
|
||||||
|
SVN_UPDATE_OP=$( svn update | tail -1 | grep Updated >/dev/null 2>&1 )
|
||||||
|
SVN_UPDATE_STATUS=$?
|
||||||
|
else
|
||||||
|
cd $R_PKGS_SVN_BASE_DIR
|
||||||
|
logger "update_r_packages: first SVN checkout."
|
||||||
|
svn co $R_PKGS_SVN_URL >/dev/null 2>&1
|
||||||
|
fi
|
||||||
|
PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_deb_pkgs.txt
|
||||||
|
R_PKGS_LIST=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_cran_pkgs.txt
|
||||||
|
R_PKGS_GITHUB=$R_PKGS_SVN_BASE_DIR/$R_PKGS_SVN_DIR/${R_PKGS_FILES_PREFIX}r_github_pkgs.txt
|
||||||
|
}
|
||||||
|
|
||||||
|
function debian_pkgs() {
|
||||||
|
if [ $DEB_PKGS_SKIP -eq 0 ] ; then
|
||||||
|
# Update the apt cache and install the packages in non interactive mode
|
||||||
|
logger "update_r_packages: Installing the debian dependencies"
|
||||||
|
if [ -z "$(find /var/cache/apt/pkgcache.bin -mmin -360)" ]; then
|
||||||
|
apt-get update -q >/dev/null 2>&1
|
||||||
|
else
|
||||||
|
logger "update_r_packages: APT cache not updated"
|
||||||
|
fi
|
||||||
|
>/var/log/update_r_debs.log
|
||||||
|
while read deb_pkg ; do
|
||||||
|
apt-get install ${deb_pkg} -q -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" >>/var/log/update_r_debs.log 2>&1
|
||||||
|
done < $PKGS_LIST
|
||||||
|
apt-get autoremove -y -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" >> /var/log/update_r_debs.log 2>&1
|
||||||
|
else
|
||||||
|
logger "update_r_packages: skipping the debian packages installation"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function remove_r_install_packages_lock_files() {
|
||||||
|
# install.packages leaves lock files around if the process crashes
|
||||||
|
rm -fr /usr/local/lib/R/site-library/00LOCK-*
|
||||||
|
}
|
||||||
|
|
||||||
|
function r_cran_pkgs() {
|
||||||
|
if [ $CRAN_PKGS_SKIP -eq 0 ] ; then
|
||||||
|
logger "update_r_packages: Installing R packages from CRAN"
|
||||||
|
for l in $( cat $R_PKGS_LIST ) ; do
|
||||||
|
pkg=$( echo $l | cut -d : -f 1 )
|
||||||
|
is_mirror_ret=
|
||||||
|
is_mirror=$( echo $l | grep ':' )
|
||||||
|
is_mirror_ret=$?
|
||||||
|
if [ $is_mirror_ret -eq 0 ] ; then
|
||||||
|
mirror=$( echo $l | cut -d : -f 2- )
|
||||||
|
else
|
||||||
|
mirror=$R_CRAN_MIRROR
|
||||||
|
fi
|
||||||
|
if [ "$ACTION" == "upgrade" ] ; then
|
||||||
|
Rscript --slave --no-save --no-restore-history -e "install.packages(pkgs='$pkg', repos=c('$mirror/'));"
|
||||||
|
else
|
||||||
|
Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { install.packages(pkgs='$pkg', repos=c('$mirror/')); }"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
else
|
||||||
|
logger "update_r_packages: skipping the R CRAN packages installation"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
function r_github_pkgs() {
|
||||||
|
if [ $GITHUB_PKGS_SKIP -eq 0 ] ; then
|
||||||
|
logger "update_r_packages: Installing R packages from Github"
|
||||||
|
for l in $( cat $R_PKGS_GITHUB ) ; do
|
||||||
|
pkg=$( echo $l | cut -d "/" -f 2 )
|
||||||
|
if [ "$ACTION" == "upgrade" ] ; then
|
||||||
|
#Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); install_github('$l');"
|
||||||
|
Rscript --slave --no-save --no-restore-history -e "require(devtools); require(methods); require(jsonlite) ; package_to_install <- '$l' ; refs <- jsonlite::read_json(sprintf('https://api.github.com/repos/%s/releases', package_to_install)) ; ref_to_install <- 'master'; if(length(refs)>0) { ref_to_install <- refs[[1]][['tag_name']] } ; devtools::install_github(package_to_install, ref = ref_to_install)"
|
||||||
|
|
||||||
|
else
|
||||||
|
#Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { require(devtools); require(methods) ; install_github('$l'); }"
|
||||||
|
Rscript --slave --no-save --no-restore-history -e "if (! ('$pkg' %in% installed.packages()[,'Package'])) { require(devtools); require(methods); require(jsonlite) ; package_to_install <- '$l' ; refs <- jsonlite::read_json(sprintf('https://api.github.com/repos/%s/releases', package_to_install)) ; ref_to_install <- 'master'; if(length(refs)>0) { ref_to_install <- refs[[1]][['tag_name']] } ; devtools::install_github(package_to_install, ref = ref_to_install) }"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
else
|
||||||
|
logger "update_r_packages: skipping the R GitHub packages installation"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
#########
|
||||||
|
# Main
|
||||||
|
#
|
||||||
|
|
||||||
|
usage
|
||||||
|
get_args
|
||||||
|
init_env
|
||||||
|
if [ $R_PKGS_FROM_SVN == 'True' ] ; then
|
||||||
|
get_data_files_from_svn
|
||||||
|
if [ $SVN_UPDATE_STATUS -ne 0 -a "$ACTION" == "install" ] ; then
|
||||||
|
logger "update_r_packages: nothing new to install from SVN, exiting"
|
||||||
|
cleanup
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
get_data_files
|
||||||
|
fi
|
||||||
|
debian_pkgs
|
||||||
|
remove_r_install_packages_lock_files
|
||||||
|
r_cran_pkgs
|
||||||
|
r_github_pkgs
|
||||||
|
cleanup
|
||||||
|
exit 0
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
7 3 * * * root /usr/local/bin/update_r_packages upgrade >/var/log/update_r_packages 2>&1
|
||||||
Loading…
Reference in New Issue