sabeel
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Pull Requests Activity

Changes for copying to Impala Cluster

This commit is contained in:
dimitrispie 2023-02-13 09:27:00 +02:00
parent d71f5672d3
commit 7b78b15c81
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh
View File

@ -24,6 +24,7 @@ function copydb() {
impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade"; impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade";
impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}"; impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}";
impala-shell -q "INVALIDATE METADATA"
echo "creating schema for ${db}" echo "creating schema for ${db}"
for i in `impala-shell -d ${db} --delimited -q "show tables"`; for i in `impala-shell -d ${db} --delimited -q "show tables"`;
do do

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -103,6 +103,7 @@
<case to="Step16_5">${wf:conf('resumeFrom') eq 'Step16_5'}</case> <case to="Step16_5">${wf:conf('resumeFrom') eq 'Step16_5'}</case>
<case to="Step19-finalize">${wf:conf('resumeFrom') eq 'Step19-finalize'}</case> <case to="Step19-finalize">${wf:conf('resumeFrom') eq 'Step19-finalize'}</case>
<case to="step20-createMonitorDB">${wf:conf('resumeFrom') eq 'step20-createMonitorDB'}</case> <case to="step20-createMonitorDB">${wf:conf('resumeFrom') eq 'step20-createMonitorDB'}</case>
<case to="step20-createMonitorDB-post">${wf:conf('resumeFrom') eq 'step20-createMonitorDB-post'}</case>
<case to="step21-createObservatoryDB-pre">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-pre'}</case> <case to="step21-createObservatoryDB-pre">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-pre'}</case>
<case to="step21-createObservatoryDB">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB'}</case> <case to="step21-createObservatoryDB">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB'}</case>
<case to="step21-createObservatoryDB-post">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-post'}</case> <case to="step21-createObservatoryDB-post">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-post'}</case>
@ -377,8 +378,7 @@
<exec>monitor-post.sh</exec> <exec>monitor-post.sh</exec>
<argument>${monitor_db_name}</argument> <argument>${monitor_db_name}</argument>
<argument>${monitor_db_shadow_name}</argument> <argument>${monitor_db_shadow_name}</argument>
<argument>${wf:appPath()}/scripts/step20-createMonitorDB.sql</argument> <file>monitor-post.sh</file>
<file>monitor.sh</file>
</shell> </shell>
<ok to="step21-createObservatoryDB-pre"/> <ok to="step21-createObservatoryDB-pre"/>
<error to="Kill"/> <error to="Kill"/>