D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 12 Projects 2 Releases 2 Wiki Activity

Prioritize the rest of the stats-queries over other tasks on the cluster, by putting them in the "analytics" queue.

This commit is contained in:
Lampros Smyrnaios 2024-07-03 19:14:25 +03:00
parent 54e11b6a43
commit aa4d7d5e20
18 changed files with 36 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step1.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
--------------------------------------------------------------
--------------------------------------------------------------
-- Stats database creation

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------------------------------------------------
------------------------------------------------------------------------------------------------
-- Tables/views from external tables/views (Fundref, Country, CountyGDP, roarmap, rndexpediture)

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
----------------------------------------------------------------
----------------------------------------------------------------
-- Post processing - Updates on main tables

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------
------------------------------------------------------
-- Additional relations

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
create view if not exists TARGET.category as select * from SOURCE.category;
create view if not exists TARGET.concept as select * from SOURCE.concept;
create view if not exists TARGET.context as select * from SOURCE.context;

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
drop database if exists TARGET cascade;
create database if not exists TARGET;

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
create table ${observatory_db_name}.result_cc_licence stored as parquet as
select r.id, coalesce(rln.count, 0) > 0 as cc_licence
from ${stats_db_name}.result r

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------
------------------------------------------------------
-- Dataset table/view and Dataset related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
--------------------------------------------------------
--------------------------------------------------------
-- Software table/view and Software related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
-- Otherresearchproduct table/view and Otherresearchproduct related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------
------------------------------------------------------
-- Project table/view and Project related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
----------------------------------------------------
----------------------------------------------------
-- Result table/view and Result related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
-- noinspection SqlNoDataSourceInspectionForFile
------------------------------------------------------------

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
----------------------------------------------------------------
----------------------------------------------------------------
-- Organization table/view and Organization related tables/views