From aedd279f78a607689ea25dd513f99cb103b7ffcb Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Wed, 12 Jul 2023 22:49:08 +0300 Subject: [PATCH 1/2] Updates Promotion DBs - Add a step for promoting the splitted monitor DBs --- .../oozie_app/updateProductionViewsMonitor.sh | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViewsMonitor.sh diff --git a/dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViewsMonitor.sh b/dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViewsMonitor.sh new file mode 100644 index 000000000..592310122 --- /dev/null +++ b/dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViewsMonitor.sh @@ -0,0 +1,44 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +export SOURCE=$1 +export PRODUCTION=$2 + +echo "Updating ${PRODUCTION} monitor database" +impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${PRODUCTION}" +impala-shell -i impala-cluster-dn1.openaire.eu -d ${PRODUCTION} -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +echo "Production monitor db ready!" + +echo "Updating ${PRODUCTION}_funded database" +impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${PRODUCTION}_funded" +impala-shell -i impala-cluster-dn1.openaire.eu -d ${PRODUCTION}_funded -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}_funded./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE}_funded -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}_funded.\1 as select * from ${SOURCE}_funded.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +echo "Production funded db ready!" + +echo "Updating ${PRODUCTION}_institutions database" +impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${PRODUCTION}_institutions" +impala-shell -i impala-cluster-dn1.openaire.eu -d ${PRODUCTION}_institutions -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}_institutions./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE}_institutions -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}_institutions.\1 as select * from ${SOURCE}_institutions.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +echo "Production insitutions db ready!" + +echo "Updating ${PRODUCTION}_ris_tail database" +impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${PRODUCTION}_ris_tail" +impala-shell -i impala-cluster-dn1.openaire.eu -d ${PRODUCTION}_ris_tail -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}_ris_tail./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE}_ris_tail -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}_ris_tail.\1 as select * from ${SOURCE}_ris_tail.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +echo "Production RIS tail db ready!" + +contexts="knowmad::other dh-ch::other enermaps::other gotriple::other neanias-atmospheric::other rural-digital-europe::other covid-19::other aurora::other neanias-space::other north-america-studies::other north-american-studies::other eutopia::other" +for i in ${contexts} +do + tmp=`echo "$i" | sed 's/'-'/'_'/g' | sed 's/'::'/'_'/g'` + impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${PRODUCTION}_${tmp}" + impala-shell -i impala-cluster-dn1.openaire.eu -d ${PRODUCTION}_${tmp} -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}_${tmp}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - + impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE}_${tmp} -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}_${tmp}.\1 as select * from ${SOURCE}_${tmp}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - + echo "Production ${tmp} db ready!" +done From 706092bc198438832949913bdc588ea6edd7b474 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Thu, 13 Jul 2023 15:48:12 +0300 Subject: [PATCH 2/2] Update updateProductionViews.sh --- .../dhp/oa/graph/stats/oozie_app/updateProductionViews.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViews.sh b/dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViews.sh index 3e510e87e..3813c7f27 100644 --- a/dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViews.sh +++ b/dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViews.sh @@ -10,7 +10,7 @@ export SOURCE=$1 export PRODUCTION=$2 echo "Updating ${PRODUCTION} database" -impala-shell -q "create database if not exists ${PRODUCTION}" -impala-shell -d ${PRODUCTION} -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}./" | sed "s/$/;/" | impala-shell -c -f - -impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -c -f - +impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${PRODUCTION}" +impala-shell -i impala-cluster-dn1.openaire.eu -d ${PRODUCTION} -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - +impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -c -f - echo "Production db ready!" \ No newline at end of file