antonis.lempesis
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 1
Code Issues Pull Requests Projects Releases Wiki Activity

computing stats after every table creation

This commit is contained in:
Antonis Lempesis 2024-10-03 10:57:10 +03:00
parent 2ed9c5504c f3c179658a
commit b6e9ffb7e3
2 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-stats-monitor-irish/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor-irish/oozie_app/scripts/buildIrishMonitorDB.sql
View File

@ -32,7 +32,7 @@ select distinct * from (
from SOURCE.result r
join SOURCE.result_projects rp on rp.id=r.id
join SOURCE.project p on p.id=rp.project
join openaire_prod_stats_monitor_ie_20231226b.irish_funders irf on irf.funder=p.funder
join TARGET.irish_funders irf on irf.funder=p.funder
union all
select r.*
from SOURCE.result r

12
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql
View File

@ -8,6 +8,18 @@ set mapred.job.queue.name=analytics; /*EOS*/
------------------------------------------------------------
------------------------------------------------------------
DROP TABLE IF EXISTS ${stats_db_name}.datasource purge; /*EOS*/
DROP TABLE IF EXISTS ${stats_db_name}.harested_datasources purge; /*EOS*/
DROP TABLE IF EXISTS ${stats_db_name}.piwik_datasource purge; /*EOS*/
create table ${stats_db_name}.harested_datasources stored as parquet as
select distinct inst.hostedby.key as d_id
from ${openaire_db_name}.result lateral view outer explode (instance) insts as inst; /*EOS*/
create table ${stats_db_name}.piwik_datasource stored as parquet as
select id, split(originalidd, '\\:')[1] as piwik_id
from ${openaire_db_name}.datasource
lateral view explode(originalid) temp as originalidd
where originalidd like "piwik:%"; /*EOS*/
create table ${stats_db_name}.harested_datasources stored as parquet as
select distinct inst.hostedby.key as d_id