You cannot select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
75 lines
2.3 KiB
Bash
75 lines
2.3 KiB
Bash
#!/bin/bash
|
|
#
|
|
# TODO:
|
|
# - print nagios friendly output into a file
|
|
# - kill rsession processes older than N days
|
|
|
|
set -e
|
|
set -o pipefail
|
|
|
|
CMD_NAME=$0
|
|
|
|
USER_N=
|
|
USER_PROCS_TO_KILL=
|
|
SPARE_MEM=1048576
|
|
USERS_SESSIONS=$( ps -edaf | grep rsession | grep -v defunct | grep -v grep | awk '{ print $10 }' | uniq )
|
|
if [ -z "$USERS_SESSIONS" ] ; then
|
|
eval logger '$CMD_NAME: There are no active sessions'
|
|
exit 0
|
|
fi
|
|
|
|
OUT_DIR=$( mktemp -d -t kill-rogue-jobs.XXXXXXXXXX )
|
|
USER_PROCS_LIST=$OUT_DIR/proclist
|
|
USER_PROCS_PARENTS=$OUT_DIR/parents
|
|
|
|
trap 'eval logger "$CMD_NAME: trap intercepted, exiting." ; cleanup' SIGHUP SIGINT SIGTERM
|
|
|
|
function cleanup() {
|
|
rm -fr "$OUT_DIR"
|
|
}
|
|
|
|
function find_rogue_processes() {
|
|
eval logger '$CMD_NAME: find_rogue_processes for user $USER_N'
|
|
ps -edaf | grep rsession | grep -v grep | grep "${USER_N}" | awk '{ print $3 }' | uniq > "$USER_PROCS_PARENTS"
|
|
pgrep -U "${USER_N}" rsession > "$USER_PROCS_LIST"
|
|
for parent in $( cat $USER_PROCS_PARENTS ) ; do
|
|
grep -v "$parent" "$USER_PROCS_LIST" > $USER_PROCS_LIST.tmp
|
|
mv $USER_PROCS_LIST.tmp "$USER_PROCS_LIST"
|
|
done
|
|
USER_PROCS_TO_KILL=$( cat "$USER_PROCS_LIST" )
|
|
}
|
|
|
|
function exterminate() {
|
|
eval logger '$CMD_NAME: exterminate killing user $USER_N processes'
|
|
for pid in $( echo "$USER_PROCS_TO_KILL" ) ; do
|
|
kill -15 "$pid"
|
|
done
|
|
}
|
|
|
|
NUM_CPUS=$( grep -c processor /proc/cpuinfo )
|
|
ALLOWED_THREADS=$(( $NUM_CPUS - 1 ))
|
|
TOTAL_MEM=$( grep MemTotal /proc/meminfo | awk '{ print $2 }' )
|
|
ALLOWED_USED_MEM=$(( $TOTAL_MEM - $SPARE_MEM ))
|
|
|
|
for USER_N in $( echo $USERS_SESSIONS ) ; do
|
|
USER_PROCS=$( ps -edaf | grep rsession | grep -v grep | grep ${USER_N} | wc -l )
|
|
USER_MEM=$( ps -eo pid,rss,vsz,args | grep rsession | grep -v grep | grep ${USER_N} | awk '{ print $2}' | paste -sd+ | bc )
|
|
if [ $USER_PROCS -gt $ALLOWED_THREADS ] || [ $USER_MEM -gt $ALLOWED_USED_MEM ] ; then
|
|
if [ $USER_PROCS -gt $ALLOWED_THREADS ] ; then
|
|
eval logger '$CMD_NAME: user $USER_N is running too many processes'
|
|
fi
|
|
if [ $USER_MEM -gt $ALLOWED_USED_MEM ] ; then
|
|
eval logger '$CMD_NAME: user $USER_N is using too much memory'
|
|
fi
|
|
find_rogue_processes
|
|
exterminate
|
|
else
|
|
eval logger '$CMD_NAME: we do not need to kill any processes for user $USER_N'
|
|
fi
|
|
done
|
|
|
|
cleanup
|
|
trap cleanup EXIT
|
|
exit 0
|
|
|