antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

Prioritize the rest of the stats-queries over other tasks on the cluster, by putting them in the "analytics" queue.

This commit is contained in:
Lampros Smyrnaios 2024-07-03 19:14:25 +03:00
parent 54e11b6a43
commit aa4d7d5e20
18 changed files with 36 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step1.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
-------------------------------------------------------------- --------------------------------------------------------------
-------------------------------------------------------------- --------------------------------------------------------------
-- Stats database creation -- Stats database creation

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------
------------------------------------------------------------------------------------------------ ------------------------------------------------------------------------------------------------
-- Tables/views from external tables/views (Fundref, Country, CountyGDP, roarmap, rndexpediture) -- Tables/views from external tables/views (Fundref, Country, CountyGDP, roarmap, rndexpediture)

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
---------------------------------------------------------------- ----------------------------------------------------------------
---------------------------------------------------------------- ----------------------------------------------------------------
-- Post processing - Updates on main tables -- Post processing - Updates on main tables

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------ ------------------------------------------------------
------------------------------------------------------ ------------------------------------------------------
-- Additional relations -- Additional relations

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
create view if not exists TARGET.category as select * from SOURCE.category; create view if not exists TARGET.category as select * from SOURCE.category;
create view if not exists TARGET.concept as select * from SOURCE.concept; create view if not exists TARGET.concept as select * from SOURCE.concept;
create view if not exists TARGET.context as select * from SOURCE.context; create view if not exists TARGET.context as select * from SOURCE.context;

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
drop database if exists TARGET cascade; drop database if exists TARGET cascade;
create database if not exists TARGET; create database if not exists TARGET;

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade; drop database if exists TARGET cascade;
create database if not exists TARGET; create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as create table TARGET.result stored as parquet as
select distinct * from ( select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade; drop database if exists TARGET cascade;
create database if not exists TARGET; create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as create table TARGET.result stored as parquet as
select distinct * from ( select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade; drop database if exists TARGET cascade;
create database if not exists TARGET; create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as create table TARGET.result stored as parquet as
select distinct * from ( select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade; drop database if exists TARGET cascade;
create database if not exists TARGET; create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as create table TARGET.result stored as parquet as
select distinct * from ( select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
create table ${observatory_db_name}.result_cc_licence stored as parquet as create table ${observatory_db_name}.result_cc_licence stored as parquet as
select r.id, coalesce(rln.count, 0) > 0 as cc_licence select r.id, coalesce(rln.count, 0) > 0 as cc_licence
from ${stats_db_name}.result r from ${stats_db_name}.result r

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------ ------------------------------------------------------
------------------------------------------------------ ------------------------------------------------------
-- Dataset table/view and Dataset related tables/views -- Dataset table/view and Dataset related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
-------------------------------------------------------- --------------------------------------------------------
-------------------------------------------------------- --------------------------------------------------------
-- Software table/view and Software related tables/views -- Software table/view and Software related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
-------------------------------------------------------------------------------- --------------------------------------------------------------------------------
-- Otherresearchproduct table/view and Otherresearchproduct related tables/views -- Otherresearchproduct table/view and Otherresearchproduct related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------ ------------------------------------------------------
------------------------------------------------------ ------------------------------------------------------
-- Project table/view and Project related tables/views -- Project table/view and Project related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
---------------------------------------------------- ----------------------------------------------------
---------------------------------------------------- ----------------------------------------------------
-- Result table/view and Result related tables/views -- Result table/view and Result related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
-- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlNoDataSourceInspectionForFile
------------------------------------------------------------ ------------------------------------------------------------

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
---------------------------------------------------------------- ----------------------------------------------------------------
---------------------------------------------------------------- ----------------------------------------------------------------
-- Organization table/view and Organization related tables/views -- Organization table/view and Organization related tables/views