D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

[stats wf] indicators across stats dbs & updates in the org ids #248

Closed
dimitris.pierrakos wants to merge 1742 commits from beta into beta2master_sept_2022
pull from: beta
merge into: D-Net:beta2master_sept_2022
Conversation 3 Commits 1742 Files Changed 1525 +3 -2
2 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7b78b15c81 - Show all commits

1
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh
View File

@ -24,6 +24,7 @@ function copydb() {
impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade"; impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade";
impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}"; impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}";
impala-shell -q "INVALIDATE METADATA"
echo "creating schema for ${db}" echo "creating schema for ${db}"
for i in `impala-shell -d ${db} --delimited -q "show tables"`; for i in `impala-shell -d ${db} --delimited -q "show tables"`;
do do

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -103,6 +103,7 @@
<case to="Step16_5">${wf:conf('resumeFrom') eq 'Step16_5'}</case> <case to="Step16_5">${wf:conf('resumeFrom') eq 'Step16_5'}</case>
<case to="Step19-finalize">${wf:conf('resumeFrom') eq 'Step19-finalize'}</case> <case to="Step19-finalize">${wf:conf('resumeFrom') eq 'Step19-finalize'}</case>
<case to="step20-createMonitorDB">${wf:conf('resumeFrom') eq 'step20-createMonitorDB'}</case> <case to="step20-createMonitorDB">${wf:conf('resumeFrom') eq 'step20-createMonitorDB'}</case>
<case to="step20-createMonitorDB-post">${wf:conf('resumeFrom') eq 'step20-createMonitorDB-post'}</case>
<case to="step21-createObservatoryDB-pre">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-pre'}</case> <case to="step21-createObservatoryDB-pre">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-pre'}</case>
<case to="step21-createObservatoryDB">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB'}</case> <case to="step21-createObservatoryDB">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB'}</case>
<case to="step21-createObservatoryDB-post">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-post'}</case> <case to="step21-createObservatoryDB-post">${wf:conf('resumeFrom') eq 'step21-createObservatoryDB-post'}</case>
@ -377,8 +378,7 @@
<exec>monitor-post.sh</exec> <exec>monitor-post.sh</exec>
<argument>${monitor_db_name}</argument> <argument>${monitor_db_name}</argument>
<argument>${monitor_db_shadow_name}</argument> <argument>${monitor_db_shadow_name}</argument>
<argument>${wf:appPath()}/scripts/step20-createMonitorDB.sql</argument> <file>monitor-post.sh</file>
<file>monitor.sh</file>
</shell> </shell>
<ok to="step21-createObservatoryDB-pre"/> <ok to="step21-createObservatoryDB-pre"/>
<error to="Kill"/> <error to="Kill"/>