D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

stats DB for monitor #99

Merged
claudio.atzori merged 15 commits from :master into master 2021-03-03 10:29:09 +01:00
Conversation 1 Commits 15 Files Changed 26 +6 -5
2 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit bcbfc052b1 - Show all commits

5
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh
View File

@ -19,6 +19,7 @@ cat step21-createMonitorDB.sql | sed s/SOURCE/$1/g | sed s/TARGET/$2/g1 | impala
echo "Impala shell finished" echo "Impala shell finished"
echo "Updating shadow monitor database" echo "Updating shadow monitor database"
impala-shell -d $SHADOW -q "show tables" --delimited | sed 's/^/drop view if exists $SHADOW./' | sed 's/$/;/' | impala-shell -f - impala-shell -q "create database if not exists ${SHADOW}"
impala-shell -d $TARGET -q "show tables" --delimited | sed 's/\(.*\)/create view $SHADOW.\1 as select * from $TARGET.\1;/' | impala-shell -f - impala-shell -d $SHADOW -q "show tables" --delimited | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -f -
impala-shell -d $TARGET -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${TARGET}.\1;/" | impala-shell -f -
echo "Shadow db ready!" echo "Shadow db ready!"

6
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -58,7 +58,7 @@
</configuration> </configuration>
</global> </global>
<start to="step21-createMonitorDB"/> <start to="Step1"/>
<kill name="Kill"> <kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message> <message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
@ -320,8 +320,8 @@
<job-tracker>${jobTracker}</job-tracker> <job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node> <name-node>${nameNode}</name-node>
<exec>monitor.sh</exec> <exec>monitor.sh</exec>
<argument>${stats-db-name}</argument> <argument>${stats_db_name}</argument>
<argument>${monitor-db-name}</argument> <argument>${monitor_db_name}</argument>
<argument>${monitor_db_shadow_name}</argument> <argument>${monitor_db_shadow_name}</argument>
<argument>${wf:appPath()}/scripts/step21-createMonitorDB.sql</argument> <argument>${wf:appPath()}/scripts/step21-createMonitorDB.sql</argument>
<file>monitor.sh</file> <file>monitor.sh</file>