spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

update promote wf to support monitor&production

This commit is contained in:
Antonis Lempesis 2021-03-12 16:42:59 +02:00
parent 60ebdf2dbe
commit 0ba0a6b9da
2 changed files with 31 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/updateProductionViews.sh
View File

@ -7,12 +7,10 @@ then
fi fi
export SOURCE=$1 export SOURCE=$1
export SHADOW=$2 export PRODUCTION=$2
echo "Updating shadow database" echo "Updating ${PRODUCTION} database"
impala-shell -d ${SOURCE} -q "invalidate metadata" impala-shell -q "create database if not exists ${PRODUCTION}"
impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/^\(.*\)/compute stats ${SOURCE}.\1;/" | impala-shell -c -f - impala-shell -d ${PRODUCTION} -q "show tables" --delimited | sed "s/^/drop view if exists ${PRODUCTION}./" | sed "s/$/;/" | impala-shell -c -f -
impala-shell -q "create database if not exists ${SHADOW}" impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${PRODUCTION}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -c -f -
impala-shell -d ${SHADOW} -q "show tables" --delimited | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -c -f - echo "Production db ready!"
impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -c -f -
echo "Shadow db ready!"

41
dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -6,7 +6,15 @@
</property> </property>
<property> <property>
<name>stats_db_production_name</name> <name>stats_db_production_name</name>
<description>the name of the production schema</description> <description>the name of the public production schema</description>
</property>
<property>
<name>monitor_db_name</name>
<description>the monitor database name</description>
</property>
<property>
<name>monitor_db_production_name</name>
<description>the name of the monitor public database</description>
</property> </property>
<property> <property>
<name>stats_tool_api_url</name> <name>stats_tool_api_url</name>
@ -48,25 +56,26 @@
</kill> </kill>
<action name="updateProductionViews"> <action name="updateProductionViews">
<hive2 xmlns="uri:oozie:hive2-action:0.1">
<jdbc-url>${hive_jdbc_url}</jdbc-url>
<script>scripts/updateProductionViews.sql</script>
<param>stats_db_name=${stats_db_name}</param>
<param>stats_db_production_name=${stats_db_production_name}</param>
</hive2>
<ok to="computeProductionStats"/>
<error to="Kill"/>
</action>
<action name="computeProductionStats">
<shell xmlns="uri:oozie:shell-action:0.1"> <shell xmlns="uri:oozie:shell-action:0.1">
<job-tracker>${jobTracker}</job-tracker> <job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node> <name-node>${nameNode}</name-node>
<exec>impala-shell.sh</exec> <exec>updateProductionViews.sh</exec>
<argument>${stats_db_name}</argument>
<argument>${stats_db_production_name}</argument> <argument>${stats_db_production_name}</argument>
<argument>computeProductionStats.sql</argument> <file>updateProductionViews.sh</file>
<argument>${wf:appPath()}/scripts/computeProductionStats.sql</argument> </shell>
<file>impala-shell.sh</file> <ok to="updateMonitorViews"/>
<error to="Kill"/>
</action>
<action name="updateMonitorViews">
<shell xmlns="uri:oozie:shell-action:0.1">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<exec>updateProductionViews.sh</exec>
<argument>${monitor_db_name}</argument>
<argument>${monitor_db_production_name}</argument>
<file>updateProductionViews.sh</file>
</shell> </shell>
<ok to="promoteCache"/> <ok to="promoteCache"/>
<error to="Kill"/> <error to="Kill"/>