Changes to separate monitor wf

dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/copyDataToImpalaCluster.sh

@@ -0,0 +1,62 @@
+export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
+export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
+if ! [ -L $link_folder ]
+then
+    rm -Rf "$link_folder"
+    ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
+fi
+
+#export HADOOP_USER_NAME="dimitris.pierrakos"
+export HADOOP_USER_NAME=$5
+
+function copydb() {
+  db=$1
+
+  # copy the databases from ocean to impala
+
+  echo "copying $db"
+  hadoop distcp -Dmapreduce.map.memory.mb=6144 -pb hdfs://nameservice1/user/hive/warehouse/${db}.db hdfs://impala-cluster-mn1.openaire.eu:8020/tmp
+
+  # change ownership to impala
+  hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/${db}.db
+
+  # create the databases
+  impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade";
+  impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}";
+
+  impala-shell -q "INVALIDATE METADATA"
+  echo "creating schema for ${db}"
+  for i in `impala-shell -d ${db} --delimited  -q "show tables"`;
+    do
+      impala-shell -d ${db} --delimited  -q "show create table $i";
+    done |  sed 's/"$/;/' | sed 's/^"//' | sed 's/[[:space:]]\date[[:space:]]/`date`/g' | impala-shell -i impala-cluster-dn1.openaire.eu -c -f -
+
+  # run the same command twice because we may have failures in the first run (due to views pointing to the same db)
+  for i in `impala-shell -d ${db} --delimited  -q "show tables"`;
+    do
+      impala-shell -d ${db} --delimited  -q "show create table $i";
+    done |  sed 's/"$/;/' | sed 's/^"//' | sed 's/[[:space:]]\date[[:space:]]/`date`/g' | impala-shell -i impala-cluster-dn1.openaire.eu -c -f -
+
+  # load the data from /tmp in the respective tables
+  echo "copying data in tables and computing stats"
+  for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited  -q "show tables"`;
+      do
+        impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "load data inpath '/tmp/${db}.db/$i' into table $i";
+        impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "compute stats $i";
+      done
+
+  # deleting the remaining directory from hdfs
+  hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -rm -R /tmp/${db}.db
+}
+
+STATS_DB=$1
+MONITOR_DB=$2
+OBSERVATORY_DB=$3
+EXT_DB=$4
+HADOOP_USER_NAME=$5
+
+copydb $EXT_DB
+copydb $STATS_DB
+copydb $MONITOR_DB
+copydb $OBSERVATORY_DB
+

dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/finalizeImpalaCluster.sh

@@ -0,0 +1,28 @@
+export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
+export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
+if ! [ -L $link_folder ]
+then
+    rm -Rf "$link_folder"
+    ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
+fi
+
+function createShadowDB() {
+  SOURCE=$1
+  SHADOW=$2
+
+  impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database ${SHADOW} CASCADE";
+  impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${SHADOW}";
+#  impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "show tables" | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f -
+  impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f -
+}
+
+STATS_DB=$1
+STATS_DB_SHADOW=$2
+MONITOR_DB=$3
+MONITOR_DB_SHADOW=$4
+OBSERVATORY_DB=$5
+OBSERVATORY_DB_SHADOW=$6
+
+createShadowDB $STATS_DB $STATS_DB_SHADOW
+createShadowDB $MONITOR_DB $MONITOR_DB_SHADOW
+createShadowDB $OBSERVATORY_DB $OBSERVATORY_DB_SHADOW

dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/updateCache.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+curl --request GET $1/cache/updateCache
+sleep 6h