From d2f9ccf9347a9828588624d109bb453795b526e5 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Mon, 20 Feb 2023 10:41:21 +0200 Subject: [PATCH] Changes to separate monitor wf --- .../oozie_app/copyDataToImpalaCluster.sh | 62 +++++++++++++++++++ .../oozie_app/finalizeImpalaCluster.sh | 28 +++++++++ .../oa/graph/monitor/oozie_app/updateCache.sh | 4 ++ 3 files changed, 94 insertions(+) create mode 100644 dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/copyDataToImpalaCluster.sh create mode 100644 dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/finalizeImpalaCluster.sh create mode 100644 dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/updateCache.sh diff --git a/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/copyDataToImpalaCluster.sh new file mode 100644 index 000000000..843877c90 --- /dev/null +++ b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/copyDataToImpalaCluster.sh @@ -0,0 +1,62 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +#export HADOOP_USER_NAME="dimitris.pierrakos" +export HADOOP_USER_NAME=$5 + +function copydb() { + db=$1 + + # copy the databases from ocean to impala + + echo "copying $db" + hadoop distcp -Dmapreduce.map.memory.mb=6144 -pb hdfs://nameservice1/user/hive/warehouse/${db}.db hdfs://impala-cluster-mn1.openaire.eu:8020/tmp + + # change ownership to impala + hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/${db}.db + + # create the databases + impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade"; + impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}"; + + impala-shell -q "INVALIDATE METADATA" + echo "creating schema for ${db}" + for i in `impala-shell -d ${db} --delimited -q "show tables"`; + do + impala-shell -d ${db} --delimited -q "show create table $i"; + done | sed 's/"$/;/' | sed 's/^"//' | sed 's/[[:space:]]\date[[:space:]]/`date`/g' | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - + + # run the same command twice because we may have failures in the first run (due to views pointing to the same db) + for i in `impala-shell -d ${db} --delimited -q "show tables"`; + do + impala-shell -d ${db} --delimited -q "show create table $i"; + done | sed 's/"$/;/' | sed 's/^"//' | sed 's/[[:space:]]\date[[:space:]]/`date`/g' | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - + + # load the data from /tmp in the respective tables + echo "copying data in tables and computing stats" + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; + do + impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "load data inpath '/tmp/${db}.db/$i' into table $i"; + impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "compute stats $i"; + done + + # deleting the remaining directory from hdfs + hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -rm -R /tmp/${db}.db +} + +STATS_DB=$1 +MONITOR_DB=$2 +OBSERVATORY_DB=$3 +EXT_DB=$4 +HADOOP_USER_NAME=$5 + +copydb $EXT_DB +copydb $STATS_DB +copydb $MONITOR_DB +copydb $OBSERVATORY_DB + diff --git a/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/finalizeImpalaCluster.sh b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/finalizeImpalaCluster.sh new file mode 100644 index 000000000..fedfa00af --- /dev/null +++ b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/finalizeImpalaCluster.sh @@ -0,0 +1,28 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +function createShadowDB() { + SOURCE=$1 + SHADOW=$2 + + impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database ${SHADOW} CASCADE"; + impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${SHADOW}"; +# impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "show tables" | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - + impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - +} + +STATS_DB=$1 +STATS_DB_SHADOW=$2 +MONITOR_DB=$3 +MONITOR_DB_SHADOW=$4 +OBSERVATORY_DB=$5 +OBSERVATORY_DB_SHADOW=$6 + +createShadowDB $STATS_DB $STATS_DB_SHADOW +createShadowDB $MONITOR_DB $MONITOR_DB_SHADOW +createShadowDB $OBSERVATORY_DB $OBSERVATORY_DB_SHADOW diff --git a/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/updateCache.sh b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/updateCache.sh new file mode 100644 index 000000000..03aa535e1 --- /dev/null +++ b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/updateCache.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +curl --request GET $1/cache/updateCache +sleep 6h \ No newline at end of file