D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

update promote wf to support monitor&production

This commit is contained in:
Antonis Lempesis 2021-03-12 16:42:59 +02:00
parent 60ebdf2dbe
commit 0ba0a6b9da
2 changed files with 31 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViews.sh
View File

@ -7,12 +7,10 @@ then
fi
export SOURCE=$1
export SHADOW=$2
export PRODUCTION=$2
echo "Updating shadow database"
impala-shell -d ${SOURCE} -q "invalidate metadata"
impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/^\(.*\)/compute stats ${SOURCE}.\1;/" | impala-shell -c -f -
impala-shell -q "create database if not exists ${SHADOW}"
impala-shell -d ${SHADOW} -q "show tables" --delimited | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -c -f -
impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -c -f -
echo "Shadow db ready!"
echo "Updating ${PRODUCTION} database"
impala-shell -q "create database if not exists ${PRODUCTION}"
impala-shell -d ${PRODUCTION} -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}./" | sed "s/$/;/" | impala-shell -c -f -
impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -c -f -
echo "Production db ready!"

41
dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -6,7 +6,15 @@
</property>
<property>
<name>stats_db_production_name</name>
<description>the name of the production schema</description>
<description>the name of the public production schema</description>
</property>
<property>
<name>monitor_db_name</name>
<description>the monitor database name</description>
</property>
<property>
<name>monitor_db_production_name</name>
<description>the name of the monitor public database</description>
</property>
<property>
<name>stats_tool_api_url</name>
@ -48,25 +56,26 @@
</kill>
<action name="updateProductionViews">
<hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url>
<script>scripts/updateProductionViews.sql</script>
<param>stats_db_name=${stats_db_name}</param>
<param>stats_db_production_name=${stats_db_production_name}</param>
</hive2>
<ok to="computeProductionStats"/>
<error to="Kill"/>
</action>
<action name="computeProductionStats">
<shell xmlns="uri:oozie:shell-action:0.1">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<exec>impala-shell.sh</exec>
<exec>updateProductionViews.sh</exec>
<argument>${stats_db_name}</argument>
<argument>${stats_db_production_name}</argument>
<argument>computeProductionStats.sql</argument>
<argument>${wf:appPath()}/scripts/computeProductionStats.sql</argument>
<file>impala-shell.sh</file>
<file>updateProductionViews.sh</file>
</shell>
<ok to="updateMonitorViews"/>
<error to="Kill"/>
</action>
<action name="updateMonitorViews">
<shell xmlns="uri:oozie:shell-action:0.1">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<exec>updateProductionViews.sh</exec>
<argument>${monitor_db_name}</argument>
<argument>${monitor_db_production_name}</argument>
<file>updateProductionViews.sh</file>
</shell>
<ok to="promoteCache"/>
<error to="Kill"/>