D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

every step is run in the analytics queue. Hardcoded for now, will make a parameter later

This commit is contained in:
Antonis Lempesis 2024-05-08 13:42:53 +03:00
parent 90a4fb3547
commit 0cada3cc8f
7 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql
View File

@ -1,3 +1,4 @@
set mapred.job.queue.name=analytics;
------------------------------------------------------
------------------------------------------------------
-- Additional relations

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics;
------------------------------------------------------
------------------------------------------------------
-- Additional relations

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics;
-------------------------------------------
--- Extra tables, mostly used by indicators

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_1-definitions.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics;
----------------------------------------------------
-- Shortcuts for various definitions in stats db ---
----------------------------------------------------

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_5.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics;
-- replace the creation of the result view to include the boolean fields from the previous tables (green, gold,
-- peer reviewed)
drop table if exists ${stats_db_name}.result_tmp;

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics;
--------------------------------------------------------------
--------------------------------------------------------------
-- Publication table/view and Publication related tables/views

1
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -368,6 +368,7 @@
${sparkClusterOpts}
${sparkResourceOpts}
${sparkApplicationOpts}
--queue analytics
</spark-opts>
<arg>--hiveMetastoreUris</arg><arg>${hive_metastore_uris}</arg>
<arg>--sql</arg><arg>eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql</arg>