antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

convert_hive_to_spark_actions #1

Merged
antonis.lempesis merged 20 commits from convert_hive_to_spark_actions into beta 2024-09-23 13:53:29 +02:00
Conversation 0 Commits 20 Files Changed 30 +13 -10
7 changed files with 13 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7b7dd32ad5 - Show all commits

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql
View File

@ -1,7 +1,7 @@
set mapred.job.queue.name=analytics;
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql
View File

@ -1,7 +1,7 @@
set mapred.job.queue.name=analytics;
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql
View File

@ -1,7 +1,7 @@
set mapred.job.queue.name=analytics;
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

4
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql
View File

@ -1,7 +1,7 @@
set mapred.job.queue.name=analytics;
drop database if exists TARGET cascade;
create database if not exists TARGET;
set mapred.job.queue.name=analytics; /*EOS*/
create table TARGET.result stored as parquet as
select distinct * from (

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql
View File

@ -139,6 +139,8 @@ from lvl1
join lvl3 on lvl3.id=lvl1.id and substr(lvl3.topic, 1, 4)=substr(lvl2.topic, 1, 4)
join lvl4 on lvl4.id=lvl1.id and substr(lvl4.topic, 1, 6)=substr(lvl3.topic, 1, 6); /*EOS*/
DROP TABLE ${stats_db_name}.result_fos_base_tmp purge; /*EOS*/
DROP TABLE IF EXISTS ${stats_db_name}.result_organization purge; /*EOS*/

2
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql
View File

@ -67,5 +67,5 @@ from ${openaire_db_name}.datasource d lateral view explode(d.collectedfrom) cfro
where d.datainfo.deletedbyinference = false and d.datainfo.invisible=false; /*EOS*/
CREATE OR REPLACE VIEW ${stats_db_name}.datasource_results AS
SELECT datasource AS id, id AS result
SELECT /*+ COALESCE(100) */ datasource AS id, id AS result
FROM ${stats_db_name}.result_datasources; /*EOS*/

3
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -150,7 +150,8 @@
</decision>
<kill name="Kill">
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
<message>Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}
${wf:actionData(wf:lastErrorNode())['stackTrace']}]</message>
</kill>
<action name="Step1">