andreas.czerniak
/
BrBETA_dnet-hadoop
forked from D-Net/dnet-hadoop
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Addedd Arts et Métiers ParisTech organization

This commit is contained in:
dimitrispie 2023-01-25 12:22:21 +02:00
parent dd70c32ad7
commit db7d625ba9
3 changed files with 7 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/monitor.sh
View File

@ -13,6 +13,6 @@ export SCRIPT_PATH=$3
echo "Getting file from " $3 echo "Getting file from " $3
hdfs dfs -copyToLocal $3 hdfs dfs -copyToLocal $3
echo "Creating monitor database" echo "Updating monitor database"
cat createMonitorDB.sql | sed s/SOURCE/$1/g | sed s/TARGET/$2/g1 | impala-shell -f - cat createMonitorDB.sql | sed s/SOURCE/$1/g | sed s/TARGET/$2/g1 | impala-shell -f -
echo "Impala shell finished" echo "Impala shell finished"

6
dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/scripts/createMonitorDB.sql
View File

@ -1,15 +1,17 @@
DROP TABLE IF EXISTS TARGET.result_new; DROP TABLE IF EXISTS TARGET.result_new;
create table TARGET.result_new stored as parquet as create table TARGET.result_new as
select distinct * from ( select distinct * from (
select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in (
-- 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork -- 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork
'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University -- 'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University
'openorgs____::57784c9e047e826fefdb1ef816120d92' --Arts et Métiers ParisTech
) )) foo; ) )) foo;
COMPUTE STATS TARGET.result_new; COMPUTE STATS TARGET.result_new;
INSERT INTO TARGET.result select * from TARGET.result_new; INSERT INTO TARGET.result select * from TARGET.result_new;
COMPUTE STATS TARGET.result;
INSERT INTO TARGET.result_citations select * from TARGET.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); INSERT INTO TARGET.result_citations select * from TARGET.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id);
COMPUTE STATS TARGET.result_citations; COMPUTE STATS TARGET.result_citations;

3
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql
View File

@ -61,7 +61,8 @@ create table TARGET.result stored as parquet as
'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje
'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan
'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork
'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University 'openorgs____::38d7097854736583dde879d12dacafca', -- Brown University
'openorgs____::57784c9e047e826fefdb1ef816120d92' --Arts et Métiers ParisTech
) )) foo; ) )) foo;
compute stats TARGET.result; compute stats TARGET.result;