From db7d625ba9b436060e8be56452215945d80efca6 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Wed, 25 Jan 2023 12:22:21 +0200 Subject: [PATCH] =?UTF-8?q?Addedd=20Arts=20et=20M=C3=A9tiers=20ParisTech?= =?UTF-8?q?=20organization?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../eu/dnetlib/dhp/oa/graph/monitor/oozie_app/monitor.sh | 2 +- .../oa/graph/monitor/oozie_app/scripts/createMonitorDB.sql | 6 ++++-- .../stats/oozie_app/scripts/step20-createMonitorDB.sql | 3 ++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/monitor.sh b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/monitor.sh index 36cfcd325..10c1ed4ca 100644 --- a/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/monitor.sh +++ b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/monitor.sh @@ -13,6 +13,6 @@ export SCRIPT_PATH=$3 echo "Getting file from " $3 hdfs dfs -copyToLocal $3 -echo "Creating monitor database" +echo "Updating monitor database" cat createMonitorDB.sql | sed s/SOURCE/$1/g | sed s/TARGET/$2/g1 | impala-shell -f - echo "Impala shell finished" diff --git a/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/scripts/createMonitorDB.sql b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/scripts/createMonitorDB.sql index 2eb95294a..265610e90 100644 --- a/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/scripts/createMonitorDB.sql +++ b/dhp-workflows/dhp-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/monitor/oozie_app/scripts/createMonitorDB.sql @@ -1,15 +1,17 @@ DROP TABLE IF EXISTS TARGET.result_new; -create table TARGET.result_new stored as parquet as +create table TARGET.result_new as select distinct * from ( select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( -- 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork - 'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University +-- 'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University + 'openorgs____::57784c9e047e826fefdb1ef816120d92' --Arts et Métiers ParisTech ) )) foo; COMPUTE STATS TARGET.result_new; INSERT INTO TARGET.result select * from TARGET.result_new; +COMPUTE STATS TARGET.result; INSERT INTO TARGET.result_citations select * from TARGET.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); COMPUTE STATS TARGET.result_citations; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index 885f7e4f7..c6b7d8ae2 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -61,7 +61,8 @@ create table TARGET.result stored as parquet as 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork - 'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University + 'openorgs____::38d7097854736583dde879d12dacafca', -- Brown University + 'openorgs____::57784c9e047e826fefdb1ef816120d92' --Arts et Métiers ParisTech ) )) foo; compute stats TARGET.result;