antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

convert_hive_to_spark_actions #1

Merged
antonis.lempesis merged 20 commits from convert_hive_to_spark_actions into beta 2024-09-23 13:53:29 +02:00
Conversation 0 Commits 20 Files Changed 30 +36
18 changed files with 36 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit aa4d7d5e20 - Show all commits

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step1.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
--------------------------------------------------------------
--------------------------------------------------------------
-- Stats database creation

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------------------------------------------------
------------------------------------------------------------------------------------------------
-- Tables/views from external tables/views (Fundref, Country, CountyGDP, roarmap, rndexpediture)

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
----------------------------------------------------------------
----------------------------------------------------------------
-- Post processing - Updates on main tables

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------
------------------------------------------------------
-- Additional relations

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
create view if not exists TARGET.category as select * from SOURCE.category;
create view if not exists TARGET.concept as select * from SOURCE.concept;
create view if not exists TARGET.context as select * from SOURCE.context;

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
drop database if exists TARGET cascade;
create database if not exists TARGET;

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql
View File

@ -1,5 +1,7 @@
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
create table ${observatory_db_name}.result_cc_licence stored as parquet as
select r.id, coalesce(rln.count, 0) > 0 as cc_licence
from ${stats_db_name}.result r

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------
------------------------------------------------------
-- Dataset table/view and Dataset related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
--------------------------------------------------------
--------------------------------------------------------
-- Software table/view and Software related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
--------------------------------------------------------------------------------
--------------------------------------------------------------------------------
-- Otherresearchproduct table/view and Otherresearchproduct related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------
------------------------------------------------------
-- Project table/view and Project related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
----------------------------------------------------
----------------------------------------------------
-- Result table/view and Result related tables/views

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
-- noinspection SqlNoDataSourceInspectionForFile
------------------------------------------------------------

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql
View File

@ -1,3 +1,5 @@
set mapred.job.queue.name=analytics; /*EOS*/
----------------------------------------------------------------
----------------------------------------------------------------
-- Organization table/view and Organization related tables/views