D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

- Fix placement of some "set mapred.job.queue.name=analytics" statements and remove their unused "/*EOS*/" indicator. 

- Add stacktrace-info to failed actions.
This commit is contained in:
Lampros Smyrnaios 2024-07-03 19:53:24 +03:00
parent 7ce051d766
commit 7b7dd32ad5
7 changed files with 13 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql
View File

@ -1,7 +1,7 @@
set mapred.job.queue.name=analytics;
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql
View File

@ -1,7 +1,7 @@
set mapred.job.queue.name=analytics;
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql
View File

@ -1,7 +1,7 @@
set mapred.job.queue.name=analytics;
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql
View File

@ -1,7 +1,7 @@
set mapred.job.queue.name=analytics;
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql
View File

@ -139,6 +139,8 @@ from lvl1
join lvl3 on lvl3.id=lvl1.id and substr(lvl3.topic, 1, 4)=substr(lvl2.topic, 1, 4)
join lvl4 on lvl4.id=lvl1.id and substr(lvl4.topic, 1, 6)=substr(lvl3.topic, 1, 6); /*EOS*/
DROP TABLE ${stats_db_name}.result_fos_base_tmp purge; /*EOS*/
DROP TABLE IF EXISTS ${stats_db_name}.result_organization purge; /*EOS*/

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql
View File

@ -67,5 +67,5 @@ from ${openaire_db_name}.datasource d lateral view explode(d.collectedfrom) cfro
where d.datainfo.deletedbyinference = false and d.datainfo.invisible=false; /*EOS*/
CREATE OR REPLACE VIEW ${stats_db_name}.datasource_results AS
SELECT datasource AS id, id AS result
SELECT /*+ COALESCE(100) */ datasource AS id, id AS result
FROM ${stats_db_name}.result_datasources; /*EOS*/

3
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -150,7 +150,8 @@
</decision>
<kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}
${wf:actionData(wf:lastErrorNode())['stackTrace']}]</message>
</kill>
<action name="Step1">