From 163b2ee2a8f27755a36de97c2f6115d27b367165 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Thu, 13 Jul 2023 15:25:00 +0300 Subject: [PATCH 01/12] Changes 1. Monitor updates 2. Bug fixes during copy to impala cluster --- .../oozie_app/config-default.xml | 30 + .../oozie_app/copyDataToImpalaCluster.sh | 75 ++ .../oozie_app/finalizeImpalaCluster.sh | 29 + .../graph/stats-monitor/oozie_app/monitor.sh | 54 ++ .../oozie_app/scripts/updateMonitorDB.sql | 138 ++++ .../oozie_app/scripts/updateMonitorDBAll.sql | 150 ++++ .../scripts/updateMonitorDB_institutions.sql | 12 + .../stats-monitor/oozie_app/workflow.xml | 110 +++ .../oozie_app/copyDataToImpalaCluster.sh | 8 +- .../stats/oozie_app/finalizeImpalaCluster.sh | 10 +- .../dhp/oa/graph/stats/oozie_app/monitor.sh | 22 +- .../graph/stats/oozie_app/scripts/step15.sql | 11 +- .../scripts/step16-createIndicatorsTables.sql | 718 +++++++++--------- .../scripts/step20-createMonitorDB.sql | 106 +-- .../scripts/step20-createMonitorDBAll.sql | 276 +++++++ .../scripts/step20-createMonitorDB_RIs.sql | 2 +- .../step20-createMonitorDB_RIs_tail.sql | 2 +- .../scripts/step20-createMonitorDB_funded.sql | 2 +- .../step20-createMonitorDB_institutions.sql | 9 +- .../scripts/step21-createObservatoryDB.sql | 38 +- .../dhp/oa/graph/stats/oozie_app/workflow.xml | 18 +- 21 files changed, 1347 insertions(+), 473 deletions(-) create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql create mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml create mode 100644 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml new file mode 100644 index 000000000..b2a1322e6 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml @@ -0,0 +1,30 @@ + + + jobTracker + ${jobTracker} + + + nameNode + ${nameNode} + + + oozie.use.system.libpath + true + + + oozie.action.sharelib.for.spark + spark2 + + + hive_metastore_uris + thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 + + + hive_jdbc_url + jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1;?spark.executor.memory=22166291558;spark.yarn.executor.memoryOverhead=3225;spark.driver.memory=15596411699;spark.yarn.driver.memoryOverhead=1228 + + + oozie.wf.workflow.notification.url + {serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status + + \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh new file mode 100644 index 000000000..1587f7152 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh @@ -0,0 +1,75 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +#export HADOOP_USER_NAME=$2 + +function copydb() { + + export HADOOP_USER="dimitris.pierrakos" + export HADOOP_USER_NAME='dimitris.pierrakos' + + db=$1 + FILE=("hive_wf_tmp_"$RANDOM) + hdfs dfs -mkdir hdfs://impala-cluster-mn1.openaire.eu:8020/tmp/$FILE/ + + # change ownership to impala +# hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/${db}.db + hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/ + + + # copy the databases from ocean to impala + echo "copying $db" + hadoop distcp -Dmapreduce.map.memory.mb=6144 -pb hdfs://nameservice1/user/hive/warehouse/${db}.db hdfs://impala-cluster-mn1.openaire.eu:8020/tmp/$FILE/ + + hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/${db}.db + + # drop tables from db + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; + do + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop table $i;"`; + done + + # drop views from db + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; + do + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop view $i;"`; + done + + # delete the database + impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade"; + + # create the databases + impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}"; + + impala-shell -q "INVALIDATE METADATA" + echo "creating schema for ${db}" + for (( k = 0; k < 5; k ++ )); do + for i in `impala-shell -d ${db} --delimited -q "show tables"`; + do + impala-shell -d ${db} --delimited -q "show create table $i"; + done | sed 's/"$/;/' | sed 's/^"//' | sed 's/[[:space:]]\date[[:space:]]/`date`/g' | impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -c -f - + done + + # load the data from /tmp in the respective tables + echo "copying data in tables and computing stats" + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; + do + impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "load data inpath '/tmp/$FILE/${db}.db/$i' into table $i"; + impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "compute stats $i"; + done + + # deleting the remaining directory from hdfs +hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -rm -R /tmp/$FILE/${db}.db +} + +MONITOR_DB=$1 +#HADOOP_USER_NAME=$2 + +copydb $MONITOR_DB'_institutions' +copydb $MONITOR_DB + diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh new file mode 100644 index 000000000..a7227e0c8 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh @@ -0,0 +1,29 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +function createShadowDB() { + SOURCE=$1 + SHADOW=$2 + + # drop views from db + for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} --delimited -q "show tables"`; + do + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "drop view $i;"`; + done + + impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database ${SHADOW} CASCADE"; + impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${SHADOW}"; +# impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "show tables" | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - + impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - +} + +MONITOR_DB=$1 +MONITOR_DB_SHADOW=$2 + +createShadowDB $MONITOR_DB'_institutions' $MONITOR_DB'_institutions_shadow' +createShadowDB $MONITOR_DB $MONITOR_DB'_shadow' diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh new file mode 100644 index 000000000..4f1889c9e --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh @@ -0,0 +1,54 @@ +export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs +export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) +if ! [ -L $link_folder ] +then + rm -Rf "$link_folder" + ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} +fi + +export SOURCE=$1 +export TARGET=$2 +export SHADOW=$3 +export SCRIPT_PATH=$4 +export SCRIPT_PATH2=$5 +export SCRIPT_PATH2=$6 + +export HIVE_OPTS="-hiveconf mapred.job.queue.name=analytics -hiveconf hive.spark.client.connect.timeout=120000ms -hiveconf hive.spark.client.server.connect.timeout=300000ms -hiveconf spark.executor.memory=19166291558 -hiveconf spark.yarn.executor.memoryOverhead=3225 -hiveconf spark.driver.memory=11596411699 -hiveconf spark.yarn.driver.memoryOverhead=1228" +export HADOOP_USER_NAME="oozie" + +echo "Getting file from " $4 +hdfs dfs -copyToLocal $4 + +echo "Getting file from " $5 +hdfs dfs -copyToLocal $5 + +echo "Getting file from " $6 +hdfs dfs -copyToLocal $6 + +#update Institutions DB +cat updateMonitorDB_institutions.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo +hive $HIVE_OPTS -f foo +cat updateMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo +hive $HIVE_OPTS -f foo + +echo "Hive shell finished" + +echo "Updating shadow monitor insitutions database" +hive -e "drop database if exists ${SHADOW}_institutions cascade" +hive -e "create database if not exists ${SHADOW}_institutions" +hive $HIVE_OPTS --database ${2}_institutions -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}_institutions.\1 as select * from ${2}_institutions.\1;/" > foo +hive -f foo +echo "Shadow db monitor insitutions ready!" + +#update Monitor DB +cat updateMonitorDBAll.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2/g1" > foo +hive $HIVE_OPTS -f foo + +echo "Hive shell finished" + +echo "Updating shadow monitor database" +hive -e "drop database if exists ${SHADOW} cascade" +hive -e "create database if not exists ${SHADOW}" +hive $HIVE_OPTS --database ${2} -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${2}.\1;/" > foo +hive -f foo +echo "Shadow db monitor insitutions ready!" diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql new file mode 100644 index 000000000..248b7e564 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql @@ -0,0 +1,138 @@ +INSERT INTO TARGET.result select * from TARGET.result_new; +ANALYZE TABLE TARGET.result COMPUTE STATISTICS; + +INSERT INTO TARGET.result_citations select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; + +INSERT INTO TARGET.result_references_oc select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; + +INSERT INTO TARGET.result_classifications select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; + +INSERT INTO TARGET.result_apc select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; + +INSERT INTO TARGET.result_concepts select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; + +INSERT INTO TARGET.result_datasources select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; + +INSERT INTO TARGET.result_fundercount select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; + +INSERT INTO TARGET.result_gold select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; + +INSERT INTO TARGET.result_greenoa select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; + +INSERT INTO TARGET.result_languages select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; + +INSERT INTO TARGET.result_licenses select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; + +INSERT INTO TARGET.result_oids select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; + +INSERT INTO TARGET.result_organization select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; + +INSERT INTO TARGET.result_peerreviewed select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; + +INSERT INTO TARGET.result_pids select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; + +INSERT INTO TARGET.result_projectcount select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; + +INSERT INTO TARGET.result_projects select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; + +INSERT INTO TARGET.result_refereed select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; + +INSERT INTO TARGET.result_sources select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; + +INSERT INTO TARGET.result_topics select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; + +INSERT INTO TARGET.result_fos select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; + +INSERT INTO TARGET.result_accessroute select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; + +create or replace view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result_new); +create or replace view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result_new); +insert into TARGET.result_result select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; +drop view TARGET.foo1; +drop view TARGET.foo2; +ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; + + +-- indicators +-- Sprint 1 ---- +INSERT INTO TARGET.indi_pub_green_oa select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_grey_lit select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_doi_from_crossref select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; +-- Sprint 2 ---- +INSERT INTO TARGET.indi_result_has_cc_licence select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_has_cc_licence_url select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_has_abstract select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_with_orcid select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; +---- Sprint 3 ---- +INSERT INTO TARGET.indi_funded_result_with_fundref select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; + +---- Sprint 4 ---- +INSERT INTO TARGET.indi_pub_diamond select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_in_transformative select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_closed_other_open select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; +---- Sprint 5 ---- +INSERT INTO TARGET.indi_result_no_of_copies select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; +---- Sprint 6 ---- +INSERT INTO TARGET.indi_pub_hybrid_oa_with_cc select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_bronze_oa select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_datasource select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_year select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_datasource_year select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; +---- Sprint 7 ---- +INSERT INTO TARGET.indi_pub_gold_oa select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_hybrid select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_has_preprint select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_in_subscribed select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_with_pid select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; +INSERT INTO TARGET.indi_impact_measures select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_interdisciplinarity select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; + +DROP TABLE IF EXISTS TARGET.result_new; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql new file mode 100644 index 000000000..478e3824e --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql @@ -0,0 +1,150 @@ +DROP TABLE IF EXISTS TARGET.result_new; + +create table TARGET.result_new as + select distinct * from ( + select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + ) )) foo; + +INSERT INTO TARGET.result select * from TARGET.result_new; +ANALYZE TABLE TARGET.result_new COMPUTE STATISTICS; + +INSERT INTO TARGET.result select * from TARGET.result_new; +ANALYZE TABLE TARGET.result COMPUTE STATISTICS; + +INSERT INTO TARGET.result_citations select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; + +INSERT INTO TARGET.result_references_oc select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; + +INSERT INTO TARGET.result_classifications select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; + +INSERT INTO TARGET.result_apc select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; + +INSERT INTO TARGET.result_concepts select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; + +INSERT INTO TARGET.result_datasources select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; + +INSERT INTO TARGET.result_fundercount select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; + +INSERT INTO TARGET.result_gold select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; + +INSERT INTO TARGET.result_greenoa select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; + +INSERT INTO TARGET.result_languages select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; + +INSERT INTO TARGET.result_licenses select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; + +INSERT INTO TARGET.result_oids select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; + +INSERT INTO TARGET.result_organization select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; + +INSERT INTO TARGET.result_peerreviewed select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; + +INSERT INTO TARGET.result_pids select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; + +INSERT INTO TARGET.result_projectcount select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; + +INSERT INTO TARGET.result_projects select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; + +INSERT INTO TARGET.result_refereed select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; + +INSERT INTO TARGET.result_sources select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; + +INSERT INTO TARGET.result_topics select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; + +INSERT INTO TARGET.result_fos select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; + +INSERT INTO TARGET.result_accessroute select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; + +create or replace view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result_new); +create or replace view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result_new); +insert into TARGET.result_result select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; +drop view TARGET.foo1; +drop view TARGET.foo2; +ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; + + +-- indicators +-- Sprint 1 ---- +INSERT INTO TARGET.indi_pub_green_oa select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_grey_lit select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_doi_from_crossref select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; +-- Sprint 2 ---- +INSERT INTO TARGET.indi_result_has_cc_licence select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_has_cc_licence_url select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_has_abstract select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_with_orcid select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; +---- Sprint 3 ---- +INSERT INTO TARGET.indi_funded_result_with_fundref select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; + +---- Sprint 4 ---- +INSERT INTO TARGET.indi_pub_diamond select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_in_transformative select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_closed_other_open select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; +---- Sprint 5 ---- +INSERT INTO TARGET.indi_result_no_of_copies select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; +---- Sprint 6 ---- +INSERT INTO TARGET.indi_pub_hybrid_oa_with_cc select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_bronze_oa select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_datasource select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_year select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_downloads_datasource_year select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); +ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; +---- Sprint 7 ---- +INSERT INTO TARGET.indi_pub_gold_oa select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_hybrid select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_has_preprint select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_in_subscribed select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; +INSERT INTO TARGET.indi_result_with_pid select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; +INSERT INTO TARGET.indi_impact_measures select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; +INSERT INTO TARGET.indi_pub_interdisciplinarity select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); +ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; + +DROP TABLE IF EXISTS TARGET.result_new; diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql new file mode 100644 index 000000000..236f3733f --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS TARGET.result_new; + +create table TARGET.result_new as + select distinct * from ( + select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + ) )) foo; + +INSERT INTO TARGET.result select * from TARGET.result_new; +ANALYZE TABLE TARGET.result_new COMPUTE STATISTICS; + diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml new file mode 100644 index 000000000..7b999a843 --- /dev/null +++ b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml @@ -0,0 +1,110 @@ + + + + stats_db_name + the target stats database name + + + monitor_db_name + the target monitor db name + + + monitor_db_shadow_name + the name of the shadow monitor db + + + hive_metastore_uris + hive server metastore URIs + + + hive_jdbc_url + hive server jdbc url + + + hive_timeout + the time period, in seconds, after which Hive fails a transaction if a Hive client has not sent a hearbeat. The default value is 300 seconds. + + + hadoop_user_name + user name of the wf owner + + + + + ${jobTracker} + ${nameNode} + + + hive.metastore.uris + ${hive_metastore_uris} + + + hive.txn.timeout + ${hive_timeout} + + + mapred.job.queue.name + analytics + + + + + + + + ${wf:conf('resumeFrom') eq 'Step1-updateMonitorDB'} + ${wf:conf('resumeFrom') eq 'Step2-copyDataToImpalaCluster'} + ${wf:conf('resumeFrom') eq 'Step3-finalizeImpalaCluster'} + + + + + + Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] + + + + + ${jobTracker} + ${nameNode} + monitor.sh + ${stats_db_name} + ${monitor_db_name} + ${monitor_db_shadow_name} + ${wf:appPath()}/scripts/updateMonitorDB_institutions.sql + ${wf:appPath()}/scripts/updateMonitorDB.sql + ${wf:appPath()}/scripts/updateMonitorDBAll.sql + monitor.sh + + + + + + + + ${jobTracker} + ${nameNode} + copyDataToImpalaCluster.sh + ${monitor_db_name} + ${hadoop_user_name} + copyDataToImpalaCluster.sh + + + + + + + + ${jobTracker} + ${nameNode} + finalizeImpalaCluster.sh + ${monitor_db_name} + ${monitor_db_shadow_name} + finalizeImpalaCluster.sh + + + + + + + diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh index 87294f6e9..431978997 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/copyDataToImpalaCluster.sh @@ -24,13 +24,13 @@ function copydb() { # drop tables from db for i in `impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; do - `impala-shell -i impala-cluster-dn1.openaire.eu -d -d ${db} -q "drop table $i;"`; + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop table $i;"`; done # drop views from db for i in `impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; do - `impala-shell -i impala-cluster-dn1.openaire.eu -d -d ${db} -q "drop view $i;"`; + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop view $i;"`; done # delete the database @@ -82,12 +82,12 @@ copydb $USAGE_STATS_DB copydb $PROD_USAGE_STATS_DB copydb $EXT_DB copydb $STATS_DB -#copydb $MONITOR_DB +copydb $MONITOR_DB copydb $OBSERVATORY_DB copydb $MONITOR_DB'_funded' copydb $MONITOR_DB'_institutions' -copydb $MONITOR_DB'_RIs_tail' +copydb $MONITOR_DB'_ris_tail' contexts="knowmad::other dh-ch::other enermaps::other gotriple::other neanias-atmospheric::other rural-digital-europe::other covid-19::other aurora::other neanias-space::other north-america-studies::other north-american-studies::other eutopia::other" for i in ${contexts} diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh index 857635b6c..86a93216c 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/finalizeImpalaCluster.sh @@ -13,7 +13,7 @@ function createShadowDB() { # drop views from db for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} --delimited -q "show tables"`; do - `impala-shell -i impala-cluster-dn1.openaire.eu -d -d ${SHADOW} -q "drop view $i;"`; + `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "drop view $i;"`; done impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database ${SHADOW} CASCADE"; @@ -36,13 +36,13 @@ createShadowDB $MONITOR_DB $MONITOR_DB_SHADOW createShadowDB $OBSERVATORY_DB $OBSERVATORY_DB_SHADOW createShadowDB USAGE_STATS_DB USAGE_STATS_DB_SHADOW -createShadowDB $MONITOR_DB'_funded' $MONITOR_DB'_funded_shadow' -createShadowDB $MONITOR_DB'_institutions' $MONITOR_DB'_institutions_shadow' -createShadowDB $MONITOR_DB'_RIs_tail' $MONITOR_DB'_RIs_tail_shadow' +createShadowDB $MONITOR_DB'_funded' $MONITOR_DB_SHADOW'_shadow_funded' +createShadowDB $MONITOR_DB'_institutions' $MONITOR_DB_SHADOW'_shadow_institutions' +createShadowDB $MONITOR_DB'_ris_tail' $MONITOR_DB_SHADOW'_shadow_ris_tail' contexts="knowmad::other dh-ch::other enermaps::other gotriple::other neanias-atmospheric::other rural-digital-europe::other covid-19::other aurora::other neanias-space::other north-america-studies::other north-american-studies::other eutopia::other" for i in ${contexts} do tmp=`echo "$i" | sed 's/'-'/'_'/g' | sed 's/'::'/'_'/g'` - createShadowDB ${MONITOR_DB}'_'${tmp} ${MONITOR_DB}'_'${tmp}'_shadow' + createShadowDB ${MONITOR_DB}'_'${tmp} ${MONITOR_DB_SHADOW}'_shadow_'${tmp} done \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh index 08f4c9232..014b19c6c 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh @@ -14,6 +14,7 @@ export SCRIPT_PATH2=$5 export SCRIPT_PATH3=$6 export SCRIPT_PATH4=$7 export SCRIPT_PATH5=$8 +export SCRIPT_PATH6=$9 export HIVE_OPTS="-hiveconf mapred.job.queue.name=analytics -hiveconf hive.spark.client.connect.timeout=120000ms -hiveconf hive.spark.client.server.connect.timeout=300000ms -hiveconf spark.executor.memory=19166291558 -hiveconf spark.yarn.executor.memoryOverhead=3225 -hiveconf spark.driver.memory=11596411699 -hiveconf spark.yarn.driver.memoryOverhead=1228" export HADOOP_USER_NAME="oozie" @@ -33,12 +34,19 @@ hdfs dfs -copyToLocal $7 echo "Getting file from " $8 hdfs dfs -copyToLocal $8 +echo "Getting file from " $9 +hdfs dfs -copyToLocal $9 + + echo "Creating monitor database" +cat step20-createMonitorDBAll.sql | sed "s/SOURCE/openaire_prod_stats_20230707/g" | sed "s/TARGET/openaire_prod_stats_monitor_20230707/g1" > foo +hive $HIVE_OPTS -f foo + cat step20-createMonitorDB_funded.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_funded/g1" > foo hive $HIVE_OPTS -f foo cat step20-createMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_funded/g1" > foo hive $HIVE_OPTS -f foo -# + cat step20-createMonitorDB_institutions.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo hive $HIVE_OPTS -f foo cat step20-createMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo @@ -56,14 +64,20 @@ do hive $HIVE_OPTS -f foo done - -cat step20-createMonitorDB_RIs_tail.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_RIs_tail/g1" | sed "s/CONTEXTS/\"'knowmad::other','dh-ch::other', 'enermaps::other', 'gotriple::other', 'neanias-atmospheric::other', 'rural-digital-europe::other', 'covid-19::other', 'aurora::other', 'neanias-space::other', 'north-america-studies::other', 'north-american-studies::other', 'eutopia::other'\"/g" > foo +cat step20-createMonitorDB_RIs_tail.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_ris_tail/g1" | sed "s/CONTEXTS/\"'knowmad::other','dh-ch::other', 'enermaps::other', 'gotriple::other', 'neanias-atmospheric::other', 'rural-digital-europe::other', 'covid-19::other', 'aurora::other', 'neanias-space::other', 'north-america-studies::other', 'north-american-studies::other', 'eutopia::other'\"/g" > foo hive $HIVE_OPTS -f foo -cat step20-createMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_RIs_tail/g1" > foo +cat step20-createMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_ris_tail/g1" > foo hive $HIVE_OPTS -f foo echo "Hive shell finished" +echo "Updating shadow monitor all database" +hive -e "drop database if exists ${SHADOW} cascade" +hive -e "create database if not exists ${SHADOW}" +hive $HIVE_OPTS --database ${2} -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${2}.\1;/" > foo +hive -f foo +echo "Updated shadow monitor all database" + echo "Updating shadow monitor funded database" hive -e "drop database if exists ${SHADOW}_funded cascade" hive -e "create database if not exists ${SHADOW}_funded" diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql index 132cb482e..75e8b001b 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql @@ -37,8 +37,17 @@ select * from ${stats_db_name}.otherresearchproduct_refereed; create table if not exists ${stats_db_name}.indi_impact_measures STORED AS PARQUET as select substr(id, 4) as id, measures_ids.id impactmetric, cast(measures_ids.unit.value[0] as double) score, -cast(measures_ids.unit.value[0] as decimal(6,3)) score_dec, measures_ids.unit.value[1] class +cast(measures_ids.unit.value[0] as decimal(6,3)) score_dec, measures_ids.unit.value[1] impact_class from ${openaire_db_name}.result lateral view explode(measures) measures as measures_ids where measures_ids.id!='views' and measures_ids.id!='downloads'; ANALYZE TABLE indi_impact_measures COMPUTE STATISTICS; + +create table if not exists ${stats_db_name}.result_apc_affiliations STORED AS PARQUET as +select distinct substr(rel.target,4) id, substr(rel.source,4) organization, o.legalname.value name, +cast(rel.properties[0].value as double) apc_amount, +rel.properties[1].value apc_currency +from ${openaire_db_name}.relation rel +join ${openaire_db_name}.organization o on o.id=rel.source +join ${openaire_db_name}.result r on r.id=rel.target +where rel.subreltype = 'affiliation' and rel.datainfo.deletedbyinference = false and size(rel.properties) > 0; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index 36b34cc3c..57c381875 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -1,88 +1,88 @@ -- Sprint 1 ---- -create table if not exists indi_pub_green_oa stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_green_oa stored as parquet as select distinct p.id, coalesce(green_oa, 0) as green_oa -from publication p +from ${stats_db_name}.publication p left outer join ( select p.id, 1 as green_oa - from publication p - join result_instance ri on ri.id = p.id - join datasource on datasource.id = ri.hostedby + from ${stats_db_name}.publication p + join ${stats_db_name}.result_instance ri on ri.id = p.id + join ${stats_db_name}.datasource on datasource.id = ri.hostedby where datasource.type like '%Repository%' and (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp on p.id= tmp.id; -ANALYZE TABLE indi_pub_green_oa COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_green_oa COMPUTE STATISTICS; -create table if not exists indi_pub_grey_lit stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_grey_lit stored as parquet as select distinct p.id, coalesce(grey_lit, 0) as grey_lit -from publication p +from ${stats_db_name}.publication p left outer join ( select p.id, 1 as grey_lit - from publication p - join result_classifications rt on rt.id = p.id + from ${stats_db_name}.publication p + join ${stats_db_name}.result_classifications rt on rt.id = p.id where rt.type not in ('Article','Part of book or chapter of book','Book','Doctoral thesis','Master thesis','Data Paper', 'Thesis', 'Bachelor thesis', 'Conference object') and - not exists (select 1 from result_classifications rc where type ='Other literature type' + not exists (select 1 from ${stats_db_name}.result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id; -ANALYZE TABLE indi_pub_grey_lit COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_grey_lit COMPUTE STATISTICS; -create table if not exists indi_pub_doi_from_crossref stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_doi_from_crossref stored as parquet as select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref -from publication p +from ${stats_db_name}.publication p left outer join - (select ri.id, 1 as doi_from_crossref from result_instance ri - join datasource d on d.id = ri.collectedfrom + (select ri.id, 1 as doi_from_crossref from ${stats_db_name}.result_instance ri + join ${stats_db_name}.datasource d on d.id = ri.collectedfrom where pidtype='Digital Object Identifier' and d.name ='Crossref') tmp on tmp.id=p.id; -ANALYZE TABLE indi_pub_doi_from_crossref COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_doi_from_crossref COMPUTE STATISTICS; -- Sprint 2 ---- -create table if not exists indi_result_has_cc_licence stored as parquet as +create table if not exists ${stats_db_name}.indi_result_has_cc_licence stored as parquet as select distinct r.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license -from result r - left outer join (select r.id, license.type as lic from result r - join result_licenses as license on license.id = r.id +from ${stats_db_name}.result r +left outer join (select r.id, license.type as lic from ${stats_db_name}.result r + join ${stats_db_name}.result_licenses as license on license.id = r.id where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp on r.id= tmp.id; -ANALYZE TABLE indi_result_has_cc_licence COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_has_cc_licence COMPUTE STATISTICS; -create table if not exists indi_result_has_cc_licence_url stored as parquet as +create table if not exists ${stats_db_name}.indi_result_has_cc_licence_url stored as parquet as select distinct r.id, case when lic_host='' or lic_host is null then 0 else 1 end as has_cc_license_url -from result r +from ${stats_db_name}.result r left outer join (select r.id, lower(parse_url(license.type, "HOST")) as lic_host - from result r - join result_licenses as license on license.id = r.id + from ${stats_db_name}.result r + join ${stats_db_name}.result_licenses as license on license.id = r.id WHERE lower(parse_url(license.type, "HOST")) = "creativecommons.org") tmp on r.id= tmp.id; -ANALYZE TABLE indi_result_has_cc_licence_url COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_has_cc_licence_url COMPUTE STATISTICS; -create table if not exists indi_pub_has_abstract stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_has_abstract stored as parquet as select distinct publication.id, cast(coalesce(abstract, true) as int) has_abstract -from publication; +from ${stats_db_name}.publication; -ANALYZE TABLE indi_pub_has_abstract COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_has_abstract COMPUTE STATISTICS; -create table if not exists indi_result_with_orcid stored as parquet as +create table if not exists ${stats_db_name}.indi_result_with_orcid stored as parquet as select distinct r.id, coalesce(has_orcid, 0) as has_orcid -from result r - left outer join (select id, 1 as has_orcid from result_orcid) tmp +from ${stats_db_name}.result r + left outer join (select id, 1 as has_orcid from ${stats_db_name}.result_orcid) tmp on r.id= tmp.id; -ANALYZE TABLE indi_result_with_orcid COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_with_orcid COMPUTE STATISTICS; ---- Sprint 3 ---- -create table if not exists indi_funded_result_with_fundref stored as parquet as +create table if not exists ${stats_db_name}.indi_funded_result_with_fundref stored as parquet as select distinct r.result as id, coalesce(fundref, 0) as fundref -from project_results r - left outer join (select distinct result, 1 as fundref from project_results +from ${stats_db_name}.project_results r + left outer join (select distinct result, 1 as fundref from ${stats_db_name}.project_results where provenance='Harvested') tmp on r.result= tmp.result; -ANALYZE TABLE indi_funded_result_with_fundref COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_funded_result_with_fundref COMPUTE STATISTICS; -- create table indi_result_org_collab stored as parquet as -- select o1.organization org1, o2.organization org2, count(distinct o1.id) as collaborations @@ -92,68 +92,68 @@ ANALYZE TABLE indi_funded_result_with_fundref COMPUTE STATISTICS; -- -- compute stats indi_result_org_collab; -- -create TEMPORARY TABLE tmp AS SELECT ro.organization organization, ro.id, o.name from result_organization ro -join organization o on o.id=ro.organization where o.name is not null; +create TEMPORARY TABLE ${stats_db_name}.tmp AS SELECT ro.organization organization, ro.id, o.name from ${stats_db_name}.result_organization ro +join ${stats_db_name}.organization o on o.id=ro.organization where o.name is not null; -create table if not exists indi_result_org_collab stored as parquet as +create table if not exists ${stats_db_name}.indi_result_org_collab stored as parquet as select o1.organization org1, o1.name org1name1, o2.organization org2, o2.name org2name2, count(o1.id) as collaborations -from tmp as o1 -join tmp as o2 where o1.id=o2.id and o1.organization!=o2.organization and o1.name!=o2.name +from ${stats_db_name}.tmp as o1 +join ${stats_db_name}.tmp as o2 where o1.id=o2.id and o1.organization!=o2.organization and o1.name!=o2.name group by o1.organization, o2.organization, o1.name, o2.name; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_result_org_collab COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_org_collab COMPUTE STATISTICS; -create TEMPORARY TABLE tmp AS -select distinct ro.organization organization, ro.id, o.name, o.country from result_organization ro -join organization o on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; +create TEMPORARY TABLE ${stats_db_name}.tmp AS +select distinct ro.organization organization, ro.id, o.name, o.country from ${stats_db_name}.result_organization ro +join ${stats_db_name}.organization o on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; -create table if not exists indi_result_org_country_collab stored as parquet as +create table if not exists ${stats_db_name}.indi_result_org_country_collab stored as parquet as select o1.organization org1,o1.name org1name1, o2.country country2, count(o1.id) as collaborations -from tmp as o1 join tmp as o2 on o1.id=o2.id +from ${stats_db_name}.tmp as o1 join ${stats_db_name}.tmp as o2 on o1.id=o2.id where o1.id=o2.id and o1.country!=o2.country group by o1.organization, o1.id, o1.name, o2.country; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_result_org_country_collab COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_org_country_collab COMPUTE STATISTICS; -create TEMPORARY TABLE tmp AS -select o.id organization, o.name, ro.project as project from organization o - join organization_projects ro on o.id=ro.id where o.name is not null; +create TEMPORARY TABLE ${stats_db_name}.tmp AS +select o.id organization, o.name, ro.project as project from ${stats_db_name}.organization o + join ${stats_db_name}.organization_projects ro on o.id=ro.id where o.name is not null; -create table if not exists indi_project_collab_org stored as parquet as +create table if not exists ${stats_db_name}.indi_project_collab_org stored as parquet as select o1.organization org1,o1.name orgname1, o2.organization org2, o2.name orgname2, count(distinct o1.project) as collaborations -from tmp as o1 - join tmp as o2 on o1.project=o2.project +from ${stats_db_name}.tmp as o1 + join ${stats_db_name}.tmp as o2 on o1.project=o2.project where o1.organization<>o2.organization and o1.name<>o2.name group by o1.name,o2.name, o1.organization, o2.organization; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_project_collab_org COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_project_collab_org COMPUTE STATISTICS; -create TEMPORARY TABLE tmp AS -select o.id organization, o.name, o.country , ro.project as project from organization o - join organization_projects ro on o.id=ro.id +create TEMPORARY TABLE ${stats_db_name}.tmp AS +select o.id organization, o.name, o.country , ro.project as project from ${stats_db_name}.organization o + join ${stats_db_name}.organization_projects ro on o.id=ro.id and o.country <> 'UNKNOWN' and o.name is not null; -create table if not exists indi_project_collab_org_country stored as parquet as +create table if not exists ${stats_db_name}.indi_project_collab_org_country stored as parquet as select o1.organization org1,o1.name org1name, o2.country country2, count(distinct o1.project) as collaborations -from tmp as o1 - join tmp as o2 on o1.project=o2.project +from ${stats_db_name}.tmp as o1 + join ${stats_db_name}.tmp as o2 on o1.project=o2.project where o1.organization<>o2.organization and o1.country<>o2.country group by o1.organization, o2.country, o1.name; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_project_collab_org_country COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_project_collab_org_country COMPUTE STATISTICS; -create table if not exists indi_funder_country_collab stored as parquet as - with tmp as (select funder, project, country from organization_projects op - join organization o on o.id=op.id - join project p on p.id=op.project +create table if not exists ${stats_db_name}.indi_funder_country_collab stored as parquet as + with tmp as (select funder, project, country from ${stats_db_name}.organization_projects op + join ${stats_db_name}.organization o on o.id=op.id + join ${stats_db_name}.project p on p.id=op.project where country <> 'UNKNOWN') select f1.funder, f1.country as country1, f2.country as country2, count(distinct f1.project) as collaborations from tmp as f1 @@ -161,104 +161,104 @@ from tmp as f1 where f1.country<>f2.country group by f1.funder, f2.country, f1.country; -ANALYZE TABLE indi_funder_country_collab COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_funder_country_collab COMPUTE STATISTICS; -create TEMPORARY TABLE tmp AS -select distinct country, ro.id as result from organization o - join result_organization ro on o.id=ro.organization +create TEMPORARY TABLE ${stats_db_name}.tmp AS +select distinct country, ro.id as result from ${stats_db_name}.organization o + join ${stats_db_name}.result_organization ro on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; -create table if not exists indi_result_country_collab stored as parquet as +create table if not exists ${stats_db_name}.indi_result_country_collab stored as parquet as select o1.country country1, o2.country country2, count(o1.result) as collaborations -from tmp as o1 - join tmp as o2 on o1.result=o2.result +from ${stats_db_name}.tmp as o1 + join ${stats_db_name}.tmp as o2 on o1.result=o2.result where o1.country<>o2.country group by o1.country, o2.country; -drop table tmp purge; +drop table ${stats_db_name}.tmp purge; -ANALYZE TABLE indi_result_country_collab COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_country_collab COMPUTE STATISTICS; ---- Sprint 4 ---- -create table if not exists indi_pub_diamond stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_diamond stored as parquet as select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal -from publication_datasources pd +from ${stats_db_name}.publication_datasources pd left outer join ( - select pd.id, 1 as in_diamond_journal from publication_datasources pd - join datasource d on d.id=pd.datasource + select pd.id, 1 as in_diamond_journal from ${stats_db_name}.publication_datasources pd + join ${stats_db_name}.datasource d on d.id=pd.datasource join STATS_EXT.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online) and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_diamond COMPUTE STATISTICS; +----ANALYZE TABLE ${stats_db_name}.indi_pub_diamond COMPUTE STATISTICS; -create table if not exists indi_pub_in_transformative stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_in_transformative stored as parquet as select distinct pd.id, coalesce(is_transformative, 0) as is_transformative -from publication pd +from ${stats_db_name}.publication pd left outer join ( - select pd.id, 1 as is_transformative from publication_datasources pd - join datasource d on d.id=pd.datasource + select pd.id, 1 as is_transformative from ${stats_db_name}.publication_datasources pd + join ${stats_db_name}.datasource d on d.id=pd.datasource join STATS_EXT.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online) and ps.is_transformative_journal=true) tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_in_transformative COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_in_transformative COMPUTE STATISTICS; -create table if not exists indi_pub_closed_other_open stored as parquet as -select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from result_instance ri +create table if not exists ${stats_db_name}.indi_pub_closed_other_open stored as parquet as +select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from ${stats_db_name}.result_instance ri left outer join - (select ri.id, 1 as pub_closed_other_open from result_instance ri - join publication p on p.id=ri.id - join datasource d on ri.hostedby=d.id + (select ri.id, 1 as pub_closed_other_open from ${stats_db_name}.result_instance ri + join ${stats_db_name}.publication p on p.id=ri.id + join ${stats_db_name}.datasource d on ri.hostedby=d.id where d.type like '%Journal%' and ri.accessright='Closed Access' and (p.bestlicence='Open Access' or p.bestlicence='Open Source')) tmp on tmp.id=ri.id; -ANALYZE TABLE indi_pub_closed_other_open COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_closed_other_open COMPUTE STATISTICS; ---- Sprint 5 ---- -create table if not exists indi_result_no_of_copies stored as parquet as -select id, count(id) as number_of_copies from result_instance group by id; +create table if not exists ${stats_db_name}.indi_result_no_of_copies stored as parquet as +select id, count(id) as number_of_copies from ${stats_db_name}.result_instance group by id; -ANALYZE TABLE indi_result_no_of_copies COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_no_of_copies COMPUTE STATISTICS; ---- Sprint 6 ---- -create table if not exists indi_pub_downloads stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_downloads stored as parquet as SELECT result_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats - join publication on result_id=id + join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id order by no_downloads desc; -ANALYZE TABLE indi_pub_downloads COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_downloads COMPUTE STATISTICS; -create table if not exists indi_pub_downloads_datasource stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_downloads_datasource stored as parquet as SELECT result_id, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats - join publication on result_id=id + join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, repository_id order by result_id; -ANALYZE TABLE indi_pub_downloads_datasource COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_datasource COMPUTE STATISTICS; -create table if not exists indi_pub_downloads_year stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_downloads_year stored as parquet as SELECT result_id, substring(us.`date`, 1,4) as `year`, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us -join publication on result_id=id where downloads>0 +join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, substring(us.`date`, 1,4); -ANALYZE TABLE indi_pub_downloads_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_year COMPUTE STATISTICS; -create table if not exists indi_pub_downloads_datasource_year stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_downloads_datasource_year stored as parquet as SELECT result_id, substring(us.`date`, 1,4) as `year`, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us -join publication on result_id=id +join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, repository_id, substring(us.`date`, 1,4); -ANALYZE TABLE indi_pub_downloads_datasource_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_datasource_year COMPUTE STATISTICS; ---- Sprint 7 ---- -create table if not exists indi_pub_gold_oa stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as WITH gold_oa AS ( SELECT issn_l, journal_is_in_doaj, @@ -284,7 +284,7 @@ create table if not exists indi_pub_gold_oa stored as parquet as id, issn_printed as issn FROM - datasource + ${stats_db_name}.datasource WHERE issn_printed IS NOT NULL UNION ALL @@ -292,7 +292,7 @@ create table if not exists indi_pub_gold_oa stored as parquet as id, issn_online as issn FROM - datasource + ${stats_db_name}.datasource WHERE issn_online IS NOT NULL or id like '%doajarticles%') as issn WHERE @@ -300,16 +300,16 @@ create table if not exists indi_pub_gold_oa stored as parquet as SELECT DISTINCT pd.id, coalesce(is_gold, 0) as is_gold FROM - publication_datasources pd + ${stats_db_name}.publication_datasources pd left outer join( - select pd.id, 1 as is_gold FROM publication_datasources pd + select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd JOIN issn on issn.id=pd.datasource JOIN gold_oa on issn.issn = gold_oa.issn) tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_gold_oa COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_gold_oa COMPUTE STATISTICS; -create table if not exists indi_pub_hybrid_oa_with_cc stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc stored as parquet as WITH hybrid_oa AS ( SELECT issn_l, journal_is_in_doaj, journal_is_oa, issn_print as issn FROM STATS_EXT.plan_s_jn @@ -322,27 +322,27 @@ create table if not exists indi_pub_hybrid_oa_with_cc stored as parquet as SELECT * FROM ( SELECT id, issn_printed as issn - FROM datasource + FROM ${stats_db_name}.datasource WHERE issn_printed IS NOT NULL UNION ALL SELECT id,issn_online as issn - FROM datasource + FROM ${stats_db_name}.datasource WHERE issn_online IS NOT NULL ) as issn WHERE LENGTH(issn) > 7) SELECT DISTINCT pd.id, coalesce(is_hybrid_oa, 0) as is_hybrid_oa -FROM publication_datasources pd +FROM ${stats_db_name}.publication_datasources pd LEFT OUTER JOIN ( - SELECT pd.id, 1 as is_hybrid_oa from publication_datasources pd - JOIN datasource d on d.id=pd.datasource + SELECT pd.id, 1 as is_hybrid_oa from ${stats_db_name}.publication_datasources pd + JOIN ${stats_db_name}.datasource d on d.id=pd.datasource JOIN issn on issn.id=pd.datasource JOIN hybrid_oa ON issn.issn = hybrid_oa.issn - JOIN indi_result_has_cc_licence cc on pd.id=cc.id - JOIN indi_pub_gold_oa ga on pd.id=ga.id + JOIN ${stats_db_name}.indi_result_has_cc_licence cc on pd.id=cc.id + JOIN ${stats_db_name}.indi_pub_gold_oa ga on pd.id=ga.id where cc.has_cc_license=1 and ga.is_gold=0) tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; -create table if not exists indi_pub_hybrid stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as WITH gold_oa AS ( SELECT issn_l, journal_is_in_doaj, @@ -370,7 +370,7 @@ create table if not exists indi_pub_hybrid stored as parquet as id, issn_printed as issn FROM - datasource + ${stats_db_name}.datasource WHERE issn_printed IS NOT NULL UNION ALL @@ -378,424 +378,398 @@ create table if not exists indi_pub_hybrid stored as parquet as id, issn_online as issn FROM - datasource + ${stats_db_name}.datasource WHERE issn_online IS NOT NULL or id like '%doajarticles%') as issn WHERE LENGTH(issn) > 7) select distinct pd.id, coalesce(is_hybrid, 0) as is_hybrid -from publication_datasources pd +from ${stats_db_name}.publication_datasources pd left outer join ( - select pd.id, 1 as is_hybrid from publication_datasources pd - join datasource d on d.id=pd.datasource + select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd + join ${stats_db_name}.datasource d on d.id=pd.datasource join issn on issn.id=pd.datasource join gold_oa on issn.issn=gold_oa.issn where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp on pd.id=tmp.id; -ANALYZE TABLE indi_pub_hybrid COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_hybrid COMPUTE STATISTICS; -create table if not exists indi_org_fairness stored as parquet as +create table if not exists ${stats_db_name}.indi_org_fairness stored as parquet as --return results with PIDs, and rich metadata group by organization with result_fair as - (select ro.organization organization, count(distinct ro.id) no_result_fair from result_organization ro - join result r on r.id=ro.id + (select ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id --join result_pids rp on r.id=rp.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 group by ro.organization), --return all results group by organization - allresults as (select organization, count(distinct ro.id) no_allresults from result_organization ro - join result r on r.id=ro.id + allresults as (select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id where cast(year as int)>2003 - group by organization) + group by ro.organization) --return results_fair/all_results select allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from allresults join result_fair on result_fair.organization=allresults.organization; -ANALYZE TABLE indi_org_fairness COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness COMPUTE STATISTICS; -create table if not exists indi_org_fairness_pub_pr stored as parquet as - with result_fair as - (select ro.organization organization, count(distinct ro.id) no_result_fair - from result_organization ro - join publication p on p.id=ro.id - join indi_pub_doi_from_crossref dc on dc.id=p.id - join indi_pub_grey_lit gl on gl.id=p.id +CREATE TEMPORARY table ${stats_db_name}.result_fair as +select ro.organization organization, count(distinct ro.id) no_result_fair + from ${stats_db_name}.result_organization ro + join ${stats_db_name}.publication p on p.id=ro.id + join ${stats_db_name}.indi_pub_doi_from_crossref dc on dc.id=p.id + join ${stats_db_name}.indi_pub_grey_lit gl on gl.id=p.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 and dc.doi_from_crossref=1 and gl.grey_lit=0 - group by ro.organization), - allresults as (select organization, count(distinct ro.id) no_allresults from result_organization ro - join publication p on p.id=ro.id + group by ro.organization; + +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as +select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.publication p on p.id=ro.id where cast(year as int)>2003 - group by organization) ---return results_fair/all_results -select allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness -from allresults - join result_fair on result_fair.organization=allresults.organization; + group by ro.organization; -ANALYZE TABLE indi_org_fairness_pub_pr COMPUTE STATISTICS; +create table if not exists ${stats_db_name}.indi_org_fairness_pub_pr stored as parquet as +select ar.organization, rf.no_result_fair/ar.no_allresults org_fairness +from ${stats_db_name}.allresults ar + join ${stats_db_name}.result_fair rf on rf.organization=ar.organization; -CREATE TEMPORARY table result_fair as - select year, ro.organization organization, count(distinct ro.id) no_result_fair from result_organization ro - join result p on p.id=ro.id +DROP table ${stats_db_name}.result_fair purge; +DROP table ${stats_db_name}.allresults purge; + +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub_pr COMPUTE STATISTICS; + +CREATE TEMPORARY table ${stats_db_name}.result_fair as + select year, ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result p on p.id=ro.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 group by ro.organization, year; -CREATE TEMPORARY TABLE allresults as select year, organization, count(distinct ro.id) no_allresults from result_organization ro - join result p on p.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as select year, ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result p on p.id=ro.id where cast(year as int)>2003 - group by organization, year; + group by ro.organization, year; -create table if not exists indi_org_fairness_pub_year stored as parquet as +create table if not exists ${stats_db_name}.indi_org_fairness_pub_year stored as parquet as select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness -from allresults - join result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; +from ${stats_db_name}.allresults + join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; -DROP table result_fair purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_fair purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_fairness_pub_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub_year COMPUTE STATISTICS; -CREATE TEMPORARY TABLE result_fair as +CREATE TEMPORARY TABLE ${stats_db_name}.result_fair as select ro.organization organization, count(distinct ro.id) no_result_fair - from result_organization ro - join result p on p.id=ro.id + from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result p on p.id=ro.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE allresults as - select organization, count(distinct ro.id) no_allresults from result_organization ro - join result p on p.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as + select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result p on p.id=ro.id where cast(year as int)>2003 - group by organization; + group by ro.organization; -create table if not exists indi_org_fairness_pub as -select allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness -from allresults join result_fair on result_fair.organization=allresults.organization; +create table if not exists ${stats_db_name}.indi_org_fairness_pub as +select ar.organization, rf.no_result_fair/ar.no_allresults org_fairness +from ${stats_db_name}.allresults ar join ${stats_db_name}.result_fair rf +on rf.organization=ar.organization; -DROP table result_fair purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_fair purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_fairness_pub COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub COMPUTE STATISTICS; -CREATE TEMPORARY TABLE result_fair as - select year, ro.organization organization, count(distinct ro.id) no_result_fair from result_organization ro - join result r on r.id=ro.id - join result_pids rp on r.id=rp.id +CREATE TEMPORARY TABLE ${stats_db_name}.result_fair as + select year, ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id + join ${stats_db_name}.result_pids rp on r.id=rp.id where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 group by ro.organization, year; -CREATE TEMPORARY TABLE allresults as - select year, organization, count(distinct ro.id) no_allresults from result_organization ro - join result r on r.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as + select year, ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id where cast(year as int)>2003 - group by organization, year; + group by ro.organization, year; -create table if not exists indi_org_fairness_year stored as parquet as +create table if not exists ${stats_db_name}.indi_org_fairness_year stored as parquet as select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness - from allresults - join result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; + from ${stats_db_name}.allresults + join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; -DROP table result_fair purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_fair purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_fairness_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_fairness_year COMPUTE STATISTICS; -CREATE TEMPORARY TABLE result_with_pid as - select year, ro.organization organization, count(distinct rp.id) no_result_with_pid from result_organization ro - join result_pids rp on rp.id=ro.id - join result r on r.id=rp.id +CREATE TEMPORARY TABLE ${stats_db_name}.result_with_pid as + select year, ro.organization, count(distinct rp.id) no_result_with_pid from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result_pids rp on rp.id=ro.id + join ${stats_db_name}.result r on r.id=rp.id where cast(year as int) >2003 group by ro.organization, year; -CREATE TEMPORARY TABLE allresults as - select year, organization, count(distinct ro.id) no_allresults from result_organization ro - join result r on r.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as + select year, ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id where cast(year as int) >2003 - group by organization, year; + group by ro.organization, year; -create table if not exists indi_org_findable_year stored as parquet as +create table if not exists ${stats_db_name}.indi_org_findable_year stored as parquet as select allresults.year, allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable -from allresults - join result_with_pid on result_with_pid.organization=allresults.organization and result_with_pid.year=allresults.year; +from ${stats_db_name}.allresults + join ${stats_db_name}.result_with_pid on result_with_pid.organization=allresults.organization and result_with_pid.year=allresults.year; -DROP table result_with_pid purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_with_pid purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_findable_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_findable_year COMPUTE STATISTICS; -CREATE TEMPORARY TABLE result_with_pid as -select ro.organization organization, count(distinct rp.id) no_result_with_pid from result_organization ro - join result_pids rp on rp.id=ro.id - join result r on r.id=rp.id +CREATE TEMPORARY TABLE ${stats_db_name}.result_with_pid as +select ro.organization, count(distinct rp.id) no_result_with_pid from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result_pids rp on rp.id=ro.id + join ${stats_db_name}.result r on r.id=rp.id where cast(year as int) >2003 group by ro.organization; -CREATE TEMPORARY TABLE allresults as -select organization, count(distinct ro.id) no_allresults from result_organization ro - join result r on r.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allresults as +select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name}.result_organization ro + join ${stats_db_name}.result r on r.id=ro.id where cast(year as int) >2003 - group by organization; + group by ro.organization; -create table if not exists indi_org_findable stored as parquet as +create table if not exists ${stats_db_name}.indi_org_findable stored as parquet as select allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable -from allresults - join result_with_pid on result_with_pid.organization=allresults.organization; +from ${stats_db_name}.allresults + join ${stats_db_name}.result_with_pid on result_with_pid.organization=allresults.organization; -DROP table result_with_pid purge; -DROP table allresults purge; +DROP table ${stats_db_name}.result_with_pid purge; +DROP table ${stats_db_name}.allresults purge; -ANALYZE TABLE indi_org_findable COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_findable COMPUTE STATISTICS; -CREATE TEMPORARY TABLE pubs_oa as -SELECT ro.organization, count(distinct r.id) no_oapubs FROM publication r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa as +SELECT ro.organization, count(distinct r.id) no_oapubs FROM ${stats_db_name}.publication r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE datasets_oa as -SELECT ro.organization, count(distinct r.id) no_oadatasets FROM dataset r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.datasets_oa as +SELECT ro.organization, count(distinct r.id) no_oadatasets FROM ${stats_db_name}.dataset r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE software_oa as -SELECT ro.organization, count(distinct r.id) no_oasoftware FROM software r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.software_oa as +SELECT ro.organization, count(distinct r.id) no_oasoftware FROM ${stats_db_name}.software r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE allpubs as -SELECT ro.organization organization, count(ro.id) no_allpubs FROM result_organization ro - join publication ps on ps.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allpubs as +SELECT ro.organization, count(ro.id) no_allpubs FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.publication ps on ps.id=ro.id where cast(ps.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE alldatasets as -SELECT ro.organization organization, count(ro.id) no_alldatasets FROM result_organization ro - join dataset ps on ps.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasets as +SELECT ro.organization, count(ro.id) no_alldatasets FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.dataset ps on ps.id=ro.id where cast(ps.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE allsoftware as -SELECT ro.organization organization, count(ro.id) no_allsoftware FROM result_organization ro - join software ps on ps.id=ro.id +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftware as +SELECT ro.organization, count(ro.id) no_allsoftware FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.software ps on ps.id=ro.id where cast(ps.year as int)>2003 group by ro.organization; -CREATE TEMPORARY TABLE allpubsshare as -select pubs_oa.organization, pubs_oa.no_oapubs/allpubs.no_allpubs p from allpubs - join pubs_oa on allpubs.organization=pubs_oa.organization; +CREATE TEMPORARY TABLE ${stats_db_name}.allpubsshare as +select pubs_oa.organization, pubs_oa.no_oapubs/allpubs.no_allpubs p from ${stats_db_name}.allpubs + join ${stats_db_name}.pubs_oa on allpubs.organization=pubs_oa.organization; -CREATE TEMPORARY TABLE alldatasetssshare as +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasetssshare as select datasets_oa.organization, datasets_oa.no_oadatasets/alldatasets.no_alldatasets d - from alldatasets - join datasets_oa on alldatasets.organization=datasets_oa.organization; + from ${stats_db_name}.alldatasets + join ${stats_db_name}.datasets_oa on alldatasets.organization=datasets_oa.organization; -CREATE TEMPORARY TABLE allsoftwaresshare as +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftwaresshare as select software_oa.organization, software_oa.no_oasoftware/allsoftware.no_allsoftware s - from allsoftware - join software_oa on allsoftware.organization=software_oa.organization; + from ${stats_db_name}.allsoftware + join ${stats_db_name}.software_oa on allsoftware.organization=software_oa.organization; -create table if not exists indi_org_openess stored as parquet as +create table if not exists ${stats_db_name}.indi_org_openess stored as parquet as select allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) +(case when d is null then 0 else 1 end)) - org_openess FROM allpubsshare + org_openess FROM ${stats_db_name}.allpubsshare left outer join (select organization,d from - alldatasetssshare) tmp1 + ${stats_db_name}.alldatasetssshare) tmp1 on tmp1.organization=allpubsshare.organization left outer join (select organization,s from - allsoftwaresshare) tmp2 + ${stats_db_name}.allsoftwaresshare) tmp2 on tmp2.organization=allpubsshare.organization; -DROP TABLE pubs_oa purge; -DROP TABLE datasets_oa purge; -DROP TABLE software_oa purge; -DROP TABLE allpubs purge; -DROP TABLE alldatasets purge; -DROP TABLE allsoftware purge; -DROP TABLE allpubsshare purge; -DROP TABLE alldatasetssshare purge; -DROP TABLE allsoftwaresshare purge; +DROP TABLE ${stats_db_name}.pubs_oa purge; +DROP TABLE ${stats_db_name}.datasets_oa purge; +DROP TABLE ${stats_db_name}.software_oa purge; +DROP TABLE ${stats_db_name}.allpubs purge; +DROP TABLE ${stats_db_name}.alldatasets purge; +DROP TABLE ${stats_db_name}.allsoftware purge; +DROP TABLE ${stats_db_name}.allpubsshare purge; +DROP TABLE ${stats_db_name}.alldatasetssshare purge; +DROP TABLE ${stats_db_name}.allsoftwaresshare purge; -ANALYZE TABLE indi_org_openess COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_openess COMPUTE STATISTICS; -CREATE TEMPORARY TABLE pubs_oa AS -SELECT r.year, ro.organization, count(distinct r.id) no_oapubs FROM publication r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa AS +SELECT r.year, ro.organization, count(distinct r.id) no_oapubs FROM ${stats_db_name}.publication r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization,r.year; -CREATE TEMPORARY TABLE datasets_oa AS -SELECT r.year,ro.organization, count(distinct r.id) no_oadatasets FROM dataset r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.datasets_oa AS +SELECT r.year,ro.organization, count(distinct r.id) no_oadatasets FROM ${stats_db_name}.dataset r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization, r.year; -CREATE TEMPORARY TABLE software_oa AS -SELECT r.year,ro.organization, count(distinct r.id) no_oasoftware FROM software r - join result_organization ro on ro.id=r.id - join result_instance ri on ri.id=r.id +CREATE TEMPORARY TABLE ${stats_db_name}.software_oa AS +SELECT r.year,ro.organization, count(distinct r.id) no_oasoftware FROM ${stats_db_name}.software r + join ${stats_db_name}.result_organization ro on ro.id=r.id + join ${stats_db_name}.result_instance ri on ri.id=r.id where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') and cast(r.year as int)>2003 group by ro.organization, r.year; -CREATE TEMPORARY TABLE allpubs as -SELECT p.year,ro.organization organization, count(ro.id) no_allpubs FROM result_organization ro - join publication p on p.id=ro.id where cast(p.year as int)>2003 +CREATE TEMPORARY TABLE ${stats_db_name}.allpubs as +SELECT p.year,ro.organization organization, count(ro.id) no_allpubs FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.publication p on p.id=ro.id where cast(p.year as int)>2003 group by ro.organization, p.year; -CREATE TEMPORARY TABLE alldatasets as -SELECT d.year, ro.organization organization, count(ro.id) no_alldatasets FROM result_organization ro - join dataset d on d.id=ro.id where cast(d.year as int)>2003 +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasets as +SELECT d.year, ro.organization organization, count(ro.id) no_alldatasets FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.dataset d on d.id=ro.id where cast(d.year as int)>2003 group by ro.organization, d.year; -CREATE TEMPORARY TABLE allsoftware as -SELECT s.year,ro.organization organization, count(ro.id) no_allsoftware FROM result_organization ro - join software s on s.id=ro.id where cast(s.year as int)>2003 +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftware as +SELECT s.year,ro.organization organization, count(ro.id) no_allsoftware FROM ${stats_db_name}.result_organization ro + join ${stats_db_name}.software s on s.id=ro.id where cast(s.year as int)>2003 group by ro.organization, s.year; -CREATE TEMPORARY TABLE allpubsshare as -select allpubs.year, pubs_oa.organization, pubs_oa.no_oapubs/allpubs.no_allpubs p from allpubs - join pubs_oa on allpubs.organization=pubs_oa.organization where cast(allpubs.year as INT)=cast(pubs_oa.year as int); +CREATE TEMPORARY TABLE ${stats_db_name}.allpubsshare as +select allpubs.year, pubs_oa.organization, pubs_oa.no_oapubs/allpubs.no_allpubs p from ${stats_db_name}.allpubs + join ${stats_db_name}.pubs_oa on allpubs.organization=pubs_oa.organization where cast(allpubs.year as INT)=cast(pubs_oa.year as int); -CREATE TEMPORARY TABLE alldatasetssshare as +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasetssshare as select alldatasets.year, datasets_oa.organization, datasets_oa.no_oadatasets/alldatasets.no_alldatasets d - from alldatasets - join datasets_oa on alldatasets.organization=datasets_oa.organization where cast(alldatasets.year as INT)=cast(datasets_oa.year as int); + from ${stats_db_name}.alldatasets + join ${stats_db_name}.datasets_oa on alldatasets.organization=datasets_oa.organization where cast(alldatasets.year as INT)=cast(datasets_oa.year as int); -CREATE TEMPORARY TABLE allsoftwaresshare as +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftwaresshare as select allsoftware.year, software_oa.organization, software_oa.no_oasoftware/allsoftware.no_allsoftware s - from allsoftware - join software_oa on allsoftware.organization=software_oa.organization where cast(allsoftware.year as INT)=cast(software_oa.year as int); + from ${stats_db_name}.allsoftware + join ${stats_db_name}.software_oa on allsoftware.organization=software_oa.organization where cast(allsoftware.year as INT)=cast(software_oa.year as int); -create table if not exists indi_org_openess_year stored as parquet as +create table if not exists ${stats_db_name}.indi_org_openess_year stored as parquet as select allpubsshare.year, allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) +(case when d is null then 0 else 1 end)) - org_openess FROM allpubsshare + org_openess FROM ${stats_db_name}.allpubsshare left outer join (select year, organization,d from - alldatasetssshare) tmp1 + ${stats_db_name}.alldatasetssshare) tmp1 on tmp1.organization=allpubsshare.organization and tmp1.year=allpubsshare.year left outer join (select year, organization,s from - allsoftwaresshare) tmp2 + ${stats_db_name}.allsoftwaresshare) tmp2 on tmp2.organization=allpubsshare.organization and tmp2.year=allpubsshare.year; -DROP TABLE pubs_oa purge; -DROP TABLE datasets_oa purge; -DROP TABLE software_oa purge; -DROP TABLE allpubs purge; -DROP TABLE alldatasets purge; -DROP TABLE allsoftware purge; -DROP TABLE allpubsshare purge; -DROP TABLE alldatasetssshare purge; -DROP TABLE allsoftwaresshare purge; +DROP TABLE ${stats_db_name}.pubs_oa purge; +DROP TABLE ${stats_db_name}.datasets_oa purge; +DROP TABLE ${stats_db_name}.software_oa purge; +DROP TABLE ${stats_db_name}.allpubs purge; +DROP TABLE ${stats_db_name}.alldatasets purge; +DROP TABLE ${stats_db_name}.allsoftware purge; +DROP TABLE ${stats_db_name}.allpubsshare purge; +DROP TABLE ${stats_db_name}.alldatasetssshare purge; +DROP TABLE ${stats_db_name}.allsoftwaresshare purge; -ANALYZE TABLE indi_org_openess_year COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_org_openess_year COMPUTE STATISTICS; -create table if not exists indi_pub_has_preprint stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_has_preprint stored as parquet as select distinct p.id, coalesce(has_preprint, 0) as has_preprint -from publication_classifications p +from ${stats_db_name}.publication_classifications p left outer join ( select p.id, 1 as has_preprint - from publication_classifications p + from ${stats_db_name}.publication_classifications p where p.type='Preprint') tmp on p.id= tmp.id; -ANALYZE TABLE indi_pub_has_preprint COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_has_preprint COMPUTE STATISTICS; -create table if not exists indi_pub_in_subscribed stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_in_subscribed stored as parquet as select distinct p.id, coalesce(is_subscription, 0) as is_subscription -from publication p +from ${stats_db_name}.publication p left outer join( - select p.id, 1 as is_subscription from publication p - join indi_pub_gold_oa g on p.id=g.id - join indi_pub_hybrid h on p.id=h.id - join indi_pub_in_transformative t on p.id=t.id + select p.id, 1 as is_subscription from ${stats_db_name}.publication p + join ${stats_db_name}.indi_pub_gold_oa g on p.id=g.id + join ${stats_db_name}.indi_pub_hybrid h on p.id=h.id + join ${stats_db_name}.indi_pub_in_transformative t on p.id=t.id where g.is_gold=0 and h.is_hybrid=0 and t.is_transformative=0) tmp on p.id=tmp.id; -ANALYZE TABLE indi_pub_in_subscribed COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_in_subscribed COMPUTE STATISTICS; -create table if not exists indi_result_with_pid as +create table if not exists ${stats_db_name}.indi_result_with_pid as select distinct p.id, coalesce(result_with_pid, 0) as result_with_pid -from result p +from ${stats_db_name}.result p left outer join ( select p.id, 1 as result_with_pid - from result_pids p) tmp + from ${stats_db_name}.result_pids p) tmp on p.id= tmp.id; -ANALYZE TABLE indi_result_with_pid COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_result_with_pid COMPUTE STATISTICS; -CREATE TEMPORARY TABLE pub_fos_totals as -select rf.id, count(distinct lvl3) totals from result_fos rf +CREATE TEMPORARY TABLE ${stats_db_name}.pub_fos_totals as +select rf.id, count(distinct lvl3) totals from ${stats_db_name}.result_fos rf group by rf.id; -create table if not exists indi_pub_interdisciplinarity as +create table if not exists ${stats_db_name}.indi_pub_interdisciplinarity as select distinct p.id as id, coalesce(is_interdisciplinary, 0) as is_interdisciplinary -from pub_fos_totals p +from ${stats_db_name}.pub_fos_totals p left outer join ( -select pub_fos_totals.id, 1 as is_interdisciplinary from pub_fos_totals +select pub_fos_totals.id, 1 as is_interdisciplinary from ${stats_db_name}.pub_fos_totals where totals>1) tmp on p.id=tmp.id; -drop table pub_fos_totals purge; +drop table ${stats_db_name}.pub_fos_totals purge; -ANALYZE TABLE indi_pub_interdisciplinarity COMPUTE STATISTICS; +--ANALYZE TABLE ${stats_db_name}.indi_pub_interdisciplinarity COMPUTE STATISTICS; -create table if not exists indi_pub_bronze_oa stored as parquet as +create table if not exists ${stats_db_name}.indi_pub_bronze_oa stored as parquet as select distinct p.id, coalesce(is_bronze_oa,0) as is_bronze_oa -from publication p +from ${stats_db_name}.publication p left outer join -(select p.id, 1 as is_bronze_oa from publication p -join indi_result_has_cc_licence cc on cc.id=p.id -join indi_pub_gold_oa ga on ga.id=p.id +(select p.id, 1 as is_bronze_oa from ${stats_db_name}.publication p +join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id +join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id where cc.has_cc_license=0 and ga.is_gold=0) tmp on tmp.id=p.id; --- create table if not exists indi_pub_bronze_oa stored as parquet as --- WITH hybrid_oa AS ( --- SELECT issn_l, journal_is_in_doaj, journal_is_oa, issn_print as issn --- FROM STATS_EXT.plan_s_jn --- WHERE issn_print != "" --- UNION ALL --- SELECT issn_l, journal_is_in_doaj, journal_is_oa, issn_online as issn --- FROM STATS_EXT.plan_s_jn --- WHERE issn_online != "" and (journal_is_in_doaj = FALSE OR journal_is_oa = FALSE)), --- issn AS ( --- SELECT * --- FROM ( --- SELECT id, issn_printed as issn --- FROM datasource --- WHERE issn_printed IS NOT NULL --- UNION ALL --- SELECT id,issn_online as issn --- FROM datasource --- WHERE issn_online IS NOT NULL ) as issn --- WHERE LENGTH(issn) > 7) ---SELECT DISTINCT pd.id, coalesce(is_bronze_oa, 0) as is_bronze_oa ---FROM publication_datasources pd --- LEFT OUTER JOIN ( --- SELECT pd.id, 1 as is_bronze_oa from publication_datasources pd --- JOIN datasource d on d.id=pd.datasource --- JOIN issn on issn.id=pd.datasource --- JOIN hybrid_oa ON issn.issn = hybrid_oa.issn --- JOIN indi_result_has_cc_licence cc on pd.id=cc.id --- JOIN indi_pub_gold_oa ga on pd.id=ga.id --- JOIN indi_pub_hybrid_oa_with_cc hy on hy.id=pd.id --- where cc.has_cc_license=0 and ga.is_gold=0 and hy.is_hybrid_oa=0) tmp on pd.id=tmp.id; - -ANALYZE TABLE indi_pub_bronze_oa COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE ${stats_db_name}.indi_pub_bronze_oa COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index 9744d5aae..3eeb792c7 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -90,83 +90,83 @@ create view if not exists TARGET.totalresearchersft as select * from SOURCE.tota create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; create table TARGET.result_references_oc stored as parquet as select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; create table TARGET.result_citations_oc stored as parquet as select * from SOURCE.result_citations_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_citations_oc COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_citations_oc COMPUTE STATISTICS; create table TARGET.result_classifications stored as parquet as select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; create table TARGET.result_apc stored as parquet as select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; create table TARGET.result_concepts stored as parquet as select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; create table TARGET.result_datasources stored as parquet as select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; create table TARGET.result_fundercount stored as parquet as select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; create table TARGET.result_gold stored as parquet as select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; create table TARGET.result_greenoa stored as parquet as select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; create table TARGET.result_languages stored as parquet as select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; create table TARGET.result_licenses stored as parquet as select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; create table TARGET.licenses_normalized STORED AS PARQUET as select * from SOURCE.licenses_normalized; -ANALYZE TABLE TARGET.licenses_normalized COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.licenses_normalized COMPUTE STATISTICS; create table TARGET.result_oids stored as parquet as select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; create table TARGET.result_organization stored as parquet as select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; create table TARGET.result_peerreviewed stored as parquet as select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; create table TARGET.result_pids stored as parquet as select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; create table TARGET.result_projectcount stored as parquet as select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; create table TARGET.result_projects stored as parquet as select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; create table TARGET.result_refereed stored as parquet as select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; create table TARGET.result_sources stored as parquet as select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; create table TARGET.result_topics stored as parquet as select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; create table TARGET.result_fos stored as parquet as select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result); create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result); create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; drop view TARGET.foo1; drop view TARGET.foo2; -ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; -- datasources create view if not exists TARGET.datasource as select * from SOURCE.datasource; @@ -175,7 +175,7 @@ create view if not exists TARGET.datasource_organizations as select * from SOURC create view if not exists TARGET.datasource_sources as select * from SOURCE.datasource_sources; create table TARGET.datasource_results stored as parquet as select id as result, datasource as id from TARGET.result_datasources; -ANALYZE TABLE TARGET.datasource_results COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.datasource_results COMPUTE STATISTICS; -- organizations create view if not exists TARGET.organization as select * from SOURCE.organization; @@ -193,28 +193,28 @@ create view if not exists TARGET.project_classification as select * from SOURCE. create view if not exists TARGET.project_organization_contribution as select * from SOURCE.project_organization_contribution; create table TARGET.project_results stored as parquet as select id as result, project as id from TARGET.result_projects; -ANALYZE TABLE TARGET.project_results COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.project_results COMPUTE STATISTICS; -- indicators -- Sprint 1 ---- create table TARGET.indi_pub_green_oa stored as parquet as select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; create table TARGET.indi_pub_grey_lit stored as parquet as select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; create table TARGET.indi_pub_doi_from_crossref stored as parquet as select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; -- Sprint 2 ---- create table TARGET.indi_result_has_cc_licence stored as parquet as select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; create table TARGET.indi_result_has_cc_licence_url stored as parquet as select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; create table TARGET.indi_pub_has_abstract stored as parquet as select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; create table TARGET.indi_result_with_orcid stored as parquet as select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; ---- Sprint 3 ---- create table TARGET.indi_funded_result_with_fundref stored as parquet as select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; create view TARGET.indi_result_org_collab as select * from SOURCE.indi_result_org_collab; create view TARGET.indi_result_org_country_collab as select * from SOURCE.indi_result_org_country_collab; create view TARGET.indi_project_collab_org as select * from SOURCE.indi_project_collab_org; @@ -223,32 +223,32 @@ create view TARGET.indi_funder_country_collab as select * from SOURCE.indi_funde create view TARGET.indi_result_country_collab as select * from SOURCE.indi_result_country_collab; ---- Sprint 4 ---- create table TARGET.indi_pub_diamond stored as parquet as select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; create table TARGET.indi_pub_in_transformative stored as parquet as select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; create table TARGET.indi_pub_closed_other_open stored as parquet as select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; ---- Sprint 5 ---- create table TARGET.indi_result_no_of_copies stored as parquet as select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; ---- Sprint 6 ---- create table TARGET.indi_pub_hybrid_oa_with_cc stored as parquet as select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; create table TARGET.indi_pub_bronze_oa stored as parquet as select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; create table TARGET.indi_pub_downloads stored as parquet as select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; create table TARGET.indi_pub_downloads_datasource stored as parquet as select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; create table TARGET.indi_pub_downloads_year stored as parquet as select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; create table TARGET.indi_pub_downloads_datasource_year stored as parquet as select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; ---- Sprint 7 ---- create table TARGET.indi_pub_gold_oa stored as parquet as select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; create table TARGET.indi_pub_hybrid stored as parquet as select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; create view TARGET.indi_org_fairness as select * from SOURCE.indi_org_fairness; create view TARGET.indi_org_fairness_pub_pr as select * from SOURCE.indi_org_fairness_pub_pr; create view TARGET.indi_org_fairness_pub_year as select * from SOURCE.indi_org_fairness_pub_year; @@ -259,12 +259,14 @@ create view TARGET.indi_org_findable as select * from SOURCE.indi_org_findable; create view TARGET.indi_org_openess as select * from SOURCE.indi_org_openess; create view TARGET.indi_org_openess_year as select * from SOURCE.indi_org_openess_year; create table TARGET.indi_pub_has_preprint stored as parquet as select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; create table TARGET.indi_pub_in_subscribed stored as parquet as select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; create table TARGET.indi_result_with_pid stored as parquet as select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; create table TARGET.indi_impact_measures stored as parquet as select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; +--ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; +create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql new file mode 100644 index 000000000..a59791084 --- /dev/null +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql @@ -0,0 +1,276 @@ +drop database if exists TARGET cascade; +create database if not exists TARGET; + +create view if not exists TARGET.category as select * from SOURCE.category; +create view if not exists TARGET.concept as select * from SOURCE.concept; +create view if not exists TARGET.context as select * from SOURCE.context; +create view if not exists TARGET.country as select * from SOURCE.country; +create view if not exists TARGET.countrygdp as select * from SOURCE.countrygdp; +create view if not exists TARGET.creation_date as select * from SOURCE.creation_date; +create view if not exists TARGET.funder as select * from SOURCE.funder; +create view if not exists TARGET.fundref as select * from SOURCE.fundref; +create view if not exists TARGET.rndexpenditure as select * from SOURCE.rndexpediture; +create view if not exists TARGET.rndgdpexpenditure as select * from SOURCE.rndgdpexpenditure; +create view if not exists TARGET.doctoratestudents as select * from SOURCE.doctoratestudents; +create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; +create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; +create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; + +create table TARGET.result stored as parquet as + select distinct * from ( + select * from SOURCE.result r where exists (select 1 from SOURCE.result_projects rp join SOURCE.project p on rp.project=p.id where rp.id=r.id) + union all + select * from SOURCE.result r where exists (select 1 from SOURCE.result_concepts rc where rc.id=r.id) + union all + select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( + 'openorgs____::b84450f9864182c67b8611b5593f4250', --"Athena Research and Innovation Center In Information Communication & Knowledge Technologies', --ARC" + 'openorgs____::d41cf6bd4ab1b1362a44397e0b95c975', --National Research Council + 'openorgs____::d2a09b9d5eabb10c95f9470e172d05d2', --??? Not exists ?? + 'openorgs____::d169c7407dd417152596908d48c11460', --Masaryk University + 'openorgs____::1ec924b1759bb16d0a02f2dad8689b21', --University of Belgrade + 'openorgs____::0ae431b820e4c33db8967fbb2b919150', --University of Helsinki + 'openorgs____::759d59f05d77188faee99b7493b46805', --University of Minho + 'openorgs____::cad284878801b9465fa51a95b1d779db', --Universidad Politécnica de Madrid + 'openorgs____::eadc8da90a546e98c03f896661a2e4d4', --University of Göttingen + 'openorgs____::c0286313e36479eff8676dba9b724b40', --National and Kapodistrian University of Athens + -- 'openorgs____::c80a8243a5e5c620d7931c88d93bf17a', --Université Paris Diderot + 'openorgs____::c08634f0a6b0081c3dc6e6c93a4314f3', --Bielefeld University + 'openorgs____::6fc85e4a8f7ecaf4b0c738d010e967ea', --University of Southern Denmark + 'openorgs____::3d6122f87f9a97a99d8f6e3d73313720', --Humboldt-Universität zu Berlin + 'openorgs____::16720ada63d0fa8ca41601feae7d1aa5', --TU Darmstadt + 'openorgs____::ccc0a066b56d2cfaf90c2ae369df16f5', --KU Leuven + 'openorgs____::4c6f119632adf789746f0a057ed73e90', --University of the Western Cape + 'openorgs____::ec3665affa01aeafa28b7852c4176dbd', --Rudjer Boskovic Institute + 'openorgs____::5f31346d444a7f06a28c880fb170b0f6', --Ghent University + 'openorgs____::2dbe47117fd5409f9c61620813456632', --University of Luxembourg + 'openorgs____::6445d7758d3a40c4d997953b6632a368', --National Institute of Informatics (NII) + 'openorgs____::b77c01aa15de3675da34277d48de2ec1', -- Valencia Catholic University Saint Vincent Martyr + 'openorgs____::7fe2f66cdc43983c6b24816bfe9cf6a0', -- Unviersity of Warsaw + 'openorgs____::15e7921fc50d9aa1229a82a84429419e', -- University Of Thessaly + 'openorgs____::11f7919dadc8f8a7251af54bba60c956', -- Technical University of Crete + 'openorgs____::84f0c5f5dbb6daf42748485924efde4b', -- University of Piraeus + 'openorgs____::4ac562f0376fce3539504567649cb373', -- University of Patras + 'openorgs____::3e8d1f8c3f6cd7f418b09f1f58b4873b', -- Aristotle University of Thessaloniki + 'openorgs____::3fcef6e1c469c10f2a84b281372c9814', -- World Bank + 'openorgs____::1698a2eb1885ef8adb5a4a969e745ad3', -- École des Ponts ParisTech + 'openorgs____::e15adb13c4dadd49de4d35c39b5da93a', -- Nanyang Technological University + 'openorgs____::4b34103bde246228fcd837f5f1bf4212', -- Autonomous University of Barcelona + 'openorgs____::72ec75fcfc4e0df1a76dc4c49007fceb', -- McMaster University + 'openorgs____::51c7fc556e46381734a25a6fbc3fd398', -- University of Modena and Reggio Emilia + 'openorgs____::235d7f9ad18ecd7e6dc62ea4990cb9db', -- Bilkent University + 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje + 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan + 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork + 'openorgs____::38d7097854736583dde879d12dacafca', -- Brown University + 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech + 'openorgs____::2530baca8a15936ba2e3297f2bce2e7e', -- University of Cape Town + 'openorgs____::d11f981828c485cd23d93f7f24f24db1', -- Technological University Dublin + 'openorgs____::5e6bf8962665cdd040341171e5c631d8', -- Delft University of Technology + 'openorgs____::846cb428d3f52a445f7275561a7beb5d', -- University of Manitoba + 'openorgs____::eb391317ed0dc684aa81ac16265de041', -- Universitat Rovira i Virgili + 'openorgs____::66aa9fc2fceb271423dfabcc38752dc0', -- Lund University + 'openorgs____::3cff625a4370d51e08624cc586138b2f', -- IMT Atlantique + 'openorgs____::c0b262bd6eab819e4c994914f9c010e2', -- National Institute of Geophysics and Volcanology + 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + ) )) foo; + +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; + +create view if not exists TARGET.category as select * from SOURCE.category; +create view if not exists TARGET.concept as select * from SOURCE.concept; +create view if not exists TARGET.context as select * from SOURCE.context; +create view if not exists TARGET.country as select * from SOURCE.country; +create view if not exists TARGET.countrygdp as select * from SOURCE.countrygdp; +create view if not exists TARGET.creation_date as select * from SOURCE.creation_date; +create view if not exists TARGET.funder as select * from SOURCE.funder; +create view if not exists TARGET.fundref as select * from SOURCE.fundref; +create view if not exists TARGET.rndexpenditure as select * from SOURCE.rndexpediture; +create view if not exists TARGET.rndgdpexpenditure as select * from SOURCE.rndgdpexpenditure; +create view if not exists TARGET.doctoratestudents as select * from SOURCE.doctoratestudents; +create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; +create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; +create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; + +create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; + +create table TARGET.result_references_oc stored as parquet as select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; + +create table TARGET.result_citations_oc stored as parquet as select * from SOURCE.result_citations_oc orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_citations_oc COMPUTE STATISTICS; + +create table TARGET.result_classifications stored as parquet as select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; + +create table TARGET.result_apc stored as parquet as select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; + +create table TARGET.result_concepts stored as parquet as select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; + +create table TARGET.result_datasources stored as parquet as select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; + +create table TARGET.result_fundercount stored as parquet as select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; + +create table TARGET.result_gold stored as parquet as select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; + +create table TARGET.result_greenoa stored as parquet as select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; + +create table TARGET.result_languages stored as parquet as select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; + +create table TARGET.result_licenses stored as parquet as select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; + +create table TARGET.licenses_normalized STORED AS PARQUET as select * from SOURCE.licenses_normalized; +--ANALYZE TABLE TARGET.licenses_normalized COMPUTE STATISTICS; + +create table TARGET.result_oids stored as parquet as select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; + +create table TARGET.result_organization stored as parquet as select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; + +create table TARGET.result_peerreviewed stored as parquet as select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; + +create table TARGET.result_pids stored as parquet as select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; + +create table TARGET.result_projectcount stored as parquet as select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; + +create table TARGET.result_projects stored as parquet as select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; + +create table TARGET.result_refereed stored as parquet as select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; + +create table TARGET.result_sources stored as parquet as select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; + +create table TARGET.result_topics stored as parquet as select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; + +create table TARGET.result_fos stored as parquet as select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; + +create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; + +create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result); +create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result); +create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; +drop view TARGET.foo1; +drop view TARGET.foo2; +--ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; + +-- datasources +create view if not exists TARGET.datasource as select * from SOURCE.datasource; +create view if not exists TARGET.datasource_oids as select * from SOURCE.datasource_oids; +create view if not exists TARGET.datasource_organizations as select * from SOURCE.datasource_organizations; +create view if not exists TARGET.datasource_sources as select * from SOURCE.datasource_sources; + +create table TARGET.datasource_results stored as parquet as select id as result, datasource as id from TARGET.result_datasources; +--ANALYZE TABLE TARGET.datasource_results COMPUTE STATISTICS; + +-- organizations +create view if not exists TARGET.organization as select * from SOURCE.organization; +create view if not exists TARGET.organization_datasources as select * from SOURCE.organization_datasources; +create view if not exists TARGET.organization_pids as select * from SOURCE.organization_pids; +create view if not exists TARGET.organization_projects as select * from SOURCE.organization_projects; +create view if not exists TARGET.organization_sources as select * from SOURCE.organization_sources; + +-- projects +create view if not exists TARGET.project as select * from SOURCE.project; +create view if not exists TARGET.project_oids as select * from SOURCE.project_oids; +create view if not exists TARGET.project_organizations as select * from SOURCE.project_organizations; +create view if not exists TARGET.project_resultcount as select * from SOURCE.project_resultcount; +create view if not exists TARGET.project_classification as select * from SOURCE.project_classification; +create view if not exists TARGET.project_organization_contribution as select * from SOURCE.project_organization_contribution; + +create table TARGET.project_results stored as parquet as select id as result, project as id from TARGET.result_projects; +--ANALYZE TABLE TARGET.project_results COMPUTE STATISTICS; + +-- indicators +-- Sprint 1 ---- +create table TARGET.indi_pub_green_oa stored as parquet as select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; +create table TARGET.indi_pub_grey_lit stored as parquet as select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; +create table TARGET.indi_pub_doi_from_crossref stored as parquet as select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; +-- Sprint 2 ---- +create table TARGET.indi_result_has_cc_licence stored as parquet as select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; +create table TARGET.indi_result_has_cc_licence_url stored as parquet as select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; +create table TARGET.indi_pub_has_abstract stored as parquet as select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; +create table TARGET.indi_result_with_orcid stored as parquet as select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; +---- Sprint 3 ---- +create table TARGET.indi_funded_result_with_fundref stored as parquet as select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; +create view TARGET.indi_result_org_collab as select * from SOURCE.indi_result_org_collab; +create view TARGET.indi_result_org_country_collab as select * from SOURCE.indi_result_org_country_collab; +create view TARGET.indi_project_collab_org as select * from SOURCE.indi_project_collab_org; +create view TARGET.indi_project_collab_org_country as select * from SOURCE.indi_project_collab_org_country; +create view TARGET.indi_funder_country_collab as select * from SOURCE.indi_funder_country_collab; +create view TARGET.indi_result_country_collab as select * from SOURCE.indi_result_country_collab; +---- Sprint 4 ---- +create table TARGET.indi_pub_diamond stored as parquet as select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; +create table TARGET.indi_pub_in_transformative stored as parquet as select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; +create table TARGET.indi_pub_closed_other_open stored as parquet as select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; +---- Sprint 5 ---- +create table TARGET.indi_result_no_of_copies stored as parquet as select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; +---- Sprint 6 ---- +create table TARGET.indi_pub_hybrid_oa_with_cc stored as parquet as select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; +create table TARGET.indi_pub_bronze_oa stored as parquet as select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; +create table TARGET.indi_pub_downloads stored as parquet as select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +--ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; +create table TARGET.indi_pub_downloads_datasource stored as parquet as select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +--ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; +create table TARGET.indi_pub_downloads_year stored as parquet as select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +--ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; +create table TARGET.indi_pub_downloads_datasource_year stored as parquet as select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +--ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; +---- Sprint 7 ---- +create table TARGET.indi_pub_gold_oa stored as parquet as select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; +create table TARGET.indi_pub_hybrid stored as parquet as select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; +create view TARGET.indi_org_fairness as select * from SOURCE.indi_org_fairness; +create view TARGET.indi_org_fairness_pub_pr as select * from SOURCE.indi_org_fairness_pub_pr; +create view TARGET.indi_org_fairness_pub_year as select * from SOURCE.indi_org_fairness_pub_year; +create view TARGET.indi_org_fairness_pub as select * from SOURCE.indi_org_fairness_pub; +create view TARGET.indi_org_fairness_year as select * from SOURCE.indi_org_fairness_year; +create view TARGET.indi_org_findable_year as select * from SOURCE.indi_org_findable_year; +create view TARGET.indi_org_findable as select * from SOURCE.indi_org_findable; +create view TARGET.indi_org_openess as select * from SOURCE.indi_org_openess; +create view TARGET.indi_org_openess_year as select * from SOURCE.indi_org_openess_year; +create table TARGET.indi_pub_has_preprint stored as parquet as select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; +create table TARGET.indi_pub_in_subscribed stored as parquet as select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; +create table TARGET.indi_result_with_pid stored as parquet as select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; +create table TARGET.indi_impact_measures stored as parquet as select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; +create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; +create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); +--ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql index 92b40405d..9a9407c2d 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs.sql @@ -12,4 +12,4 @@ create table TARGET.result stored as parquet as -- join SOURCE.result where rc.id=r.id and conc.category like CONTEXT) ) foo; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql index ef6d08d79..bad18efde 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_RIs_tail.sql @@ -12,4 +12,4 @@ create table TARGET.result stored as parquet as -- join SOURCE.result where rc.id=r.id and conc.category not in (CONTEXTS)) ) foo; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql index 8d8739c74..b8d3c0242 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_funded.sql @@ -6,4 +6,4 @@ create table TARGET.result stored as parquet as select * from SOURCE.result r where exists (select 1 from SOURCE.result_projects rp join SOURCE.project p on rp.project=p.id where rp.id=r.id) ) foo; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql index 442e623cd..1f75c3cd1 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql @@ -42,7 +42,7 @@ create table TARGET.result stored as parquet as 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork - 'openorgs____::38d7097854736583dde879d12dacafca' -- Brown University + 'openorgs____::38d7097854736583dde879d12dacafca', -- Brown University 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech 'openorgs____::2530baca8a15936ba2e3297f2bce2e7e', -- University of Cape Town 'openorgs____::d11f981828c485cd23d93f7f24f24db1', -- Technological University Dublin @@ -52,7 +52,10 @@ create table TARGET.result stored as parquet as 'openorgs____::66aa9fc2fceb271423dfabcc38752dc0', -- Lund University 'openorgs____::3cff625a4370d51e08624cc586138b2f', -- IMT Atlantique 'openorgs____::c0b262bd6eab819e4c994914f9c010e2', -- National Institute of Geophysics and Volcanology - 'openorgs____::1624ff7c01bb641b91f4518539a0c28a' -- Vrije Universiteit Amsterdam + 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + ))) foo; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql index 2d7d572b3..b7e421813 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step21-createObservatoryDB.sql @@ -8,7 +8,7 @@ from ${stats_db_name}.result r group by rl.id ) rln on rln.id=r.id; -ANALYZE TABLE ${observatory_db_name}.result_cc_licence COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_cc_licence COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_country stored as parquet as select @@ -39,7 +39,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_year stored as parquet as select @@ -70,7 +70,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_year COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_year COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_year_country stored as parquet as select @@ -101,7 +101,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_year_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_year_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_datasource stored as parquet as select @@ -134,7 +134,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_datasource_country stored as parquet as select @@ -167,7 +167,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_datasource_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_organization stored as parquet as select @@ -198,7 +198,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_organization_country stored as parquet as select @@ -229,7 +229,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_organization_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_funder stored as parquet as select @@ -262,7 +262,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder COMPUTE STATISTICS; create table ${observatory_db_name}.result_affiliated_funder_country stored as parquet as select @@ -295,7 +295,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_affiliated_funder_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_country stored as parquet as select @@ -328,7 +328,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_year stored as parquet as select @@ -361,7 +361,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year; -ANALYZE TABLE ${observatory_db_name}.result_deposited_year COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_year COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_year_country stored as parquet as select @@ -394,7 +394,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, r.year, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_year_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_year_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_datasource stored as parquet as select @@ -427,7 +427,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_datasource_country stored as parquet as select @@ -460,7 +460,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, d.name, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_datasource_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_organization stored as parquet as select @@ -493,7 +493,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_organization COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_organization COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_organization_country stored as parquet as select @@ -526,7 +526,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, o.name, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_organization_country COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_organization_country COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_funder stored as parquet as select @@ -561,7 +561,7 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder; -ANALYZE TABLE ${observatory_db_name}.result_deposited_funder COMPUTE STATISTICS; +--ANALYZE TABLE ${observatory_db_name}.result_deposited_funder COMPUTE STATISTICS; create table ${observatory_db_name}.result_deposited_funder_country stored as parquet as select @@ -596,4 +596,4 @@ group by r.green, r.gold, case when rl.type is not null then true else false end case when r.access_mode in ('Open Access', 'Open Source') then true else false end, r.peer_reviewed, r.type, abstract, cc_licence, r.authors > 1, rpc.count > 1, rfc.count > 1, p.funder, c.code, c.name; -ANALYZE TABLE ${observatory_db_name}.result_deposited_funder_country COMPUTE STATISTICS; \ No newline at end of file +--ANALYZE TABLE ${observatory_db_name}.result_deposited_funder_country COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml index 2ab50fb29..c03520e48 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml @@ -317,15 +317,12 @@ - - ${jobTracker} - ${nameNode} - indicators.sh - ${stats_db_name} - ${external_stats_db_name} - ${wf:appPath()}/scripts/step16-createIndicatorsTables.sql - indicators.sh - + + ${hive_jdbc_url} + + stats_db_name=${stats_db_name} + external_stats_db_name=${external_stats_db_name} + @@ -378,6 +375,7 @@ ${wf:appPath()}/scripts/step20-createMonitorDB_institutions.sql ${wf:appPath()}/scripts/step20-createMonitorDB_RIs.sql ${wf:appPath()}/scripts/step20-createMonitorDB_RIs_tail.sql + ${wf:appPath()}/scripts/step20-createMonitorDBAll.sql monitor.sh @@ -469,7 +467,7 @@ ${usage_stats_db_shadow_name} finalizeImpalaCluster.sh - + From be4856ef35401dc7a6e969763839254e645456fb Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Mon, 17 Jul 2023 15:33:58 +0300 Subject: [PATCH 02/12] Update step15.sql --- .../eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql index 75e8b001b..d1cbde438 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql @@ -41,8 +41,6 @@ cast(measures_ids.unit.value[0] as decimal(6,3)) score_dec, measures_ids.unit.va from ${openaire_db_name}.result lateral view explode(measures) measures as measures_ids where measures_ids.id!='views' and measures_ids.id!='downloads'; -ANALYZE TABLE indi_impact_measures COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.result_apc_affiliations STORED AS PARQUET as select distinct substr(rel.target,4) id, substr(rel.source,4) organization, o.legalname.value name, cast(rel.properties[0].value as double) apc_amount, From 964c2f553e43438cedbfe44d2b1ae5e4d4d3d4f6 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Fri, 1 Sep 2023 10:57:02 +0300 Subject: [PATCH 03/12] Changes in indicators step, monitor step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - graduatedoctorates for observatory - result_apc_affiliations table - new indicators indi_is_funder_plan_s indi_funder_fairness indi_ris_fairness indi_funder_openess indi_ris_openess indi_funder_findable indi_ris_findable indi_is_project_result_after - cast year to int in composite indicators - new institutions -- Universidade Católica Portuguesa -- Iscte - Instituto Universitário de Lisboa -- Munster Technological University -- Cardiff University -- Leibniz Institute of Ecological Urban and Regional Development --- .../dhp/oa/graph/stats/oozie_app/monitor.sh | 2 +- .../graph/stats/oozie_app/scripts/step13.sql | 0 .../graph/stats/oozie_app/scripts/step15.sql | 2 +- .../stats/oozie_app/scripts/step15_5.sql | 1 + .../scripts/step16-createIndicatorsTables.sql | 358 ++++++++++++++---- .../scripts/step20-createMonitorDB.sql | 9 + .../scripts/step20-createMonitorDBAll.sql | 18 +- .../step20-createMonitorDB_institutions.sql | 25 +- .../graph/stats/oozie_app/scripts/step5.sql | 0 .../dhp/oa/graph/stats/oozie_app/workflow.xml | 2 +- 10 files changed, 324 insertions(+), 93 deletions(-) mode change 100644 => 100755 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh mode change 100644 => 100755 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql mode change 100644 => 100755 dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh old mode 100644 new mode 100755 index 014b19c6c..872456973 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/monitor.sh @@ -39,7 +39,7 @@ hdfs dfs -copyToLocal $9 echo "Creating monitor database" -cat step20-createMonitorDBAll.sql | sed "s/SOURCE/openaire_prod_stats_20230707/g" | sed "s/TARGET/openaire_prod_stats_monitor_20230707/g1" > foo +cat step20-createMonitorDBAll.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2/g1" > foo hive $HIVE_OPTS -f foo cat step20-createMonitorDB_funded.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_funded/g1" > foo diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql old mode 100644 new mode 100755 diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql index d1cbde438..4a8f81943 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql @@ -48,4 +48,4 @@ rel.properties[1].value apc_currency from ${openaire_db_name}.relation rel join ${openaire_db_name}.organization o on o.id=rel.source join ${openaire_db_name}.result r on r.id=rel.target -where rel.subreltype = 'affiliation' and rel.datainfo.deletedbyinference = false and size(rel.properties) > 0; +where rel.subreltype = 'affiliation' and rel.datainfo.deletedbyinference = false and size(rel.properties)>0; diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql index f39ff2afd..615f523ce 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15_5.sql @@ -35,6 +35,7 @@ create or replace view ${stats_db_name}.doctoratestudents as select * from stats create or replace view ${stats_db_name}.totalresearchers as select * from stats_ext.totalresearchers; create or replace view ${stats_db_name}.totalresearchersft as select * from stats_ext.totalresearchersft; create or replace view ${stats_db_name}.hrrst as select * from stats_ext.hrrst; +create or replace view ${stats_db_name}.graduatedoctorates as select * from stats_ext.graduatedoctorates; create table if not exists ${stats_db_name}.result_instance stored as parquet as select distinct r.* diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index 57c381875..1c80f6757 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -12,8 +12,6 @@ from ${stats_db_name}.publication p or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp on p.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_green_oa COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_grey_lit stored as parquet as select distinct p.id, coalesce(grey_lit, 0) as grey_lit from ${stats_db_name}.publication p @@ -25,8 +23,6 @@ from ${stats_db_name}.publication p not exists (select 1 from ${stats_db_name}.result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_grey_lit COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_doi_from_crossref stored as parquet as select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref from ${stats_db_name}.publication p @@ -36,8 +32,6 @@ from ${stats_db_name}.publication p where pidtype='Digital Object Identifier' and d.name ='Crossref') tmp on tmp.id=p.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_doi_from_crossref COMPUTE STATISTICS; - -- Sprint 2 ---- create table if not exists ${stats_db_name}.indi_result_has_cc_licence stored as parquet as select distinct r.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license @@ -47,8 +41,6 @@ left outer join (select r.id, license.type as lic from ${stats_db_name}.result r where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp on r.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_result_has_cc_licence COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_result_has_cc_licence_url stored as parquet as select distinct r.id, case when lic_host='' or lic_host is null then 0 else 1 end as has_cc_license_url from ${stats_db_name}.result r @@ -58,22 +50,16 @@ from ${stats_db_name}.result r WHERE lower(parse_url(license.type, "HOST")) = "creativecommons.org") tmp on r.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_result_has_cc_licence_url COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_has_abstract stored as parquet as select distinct publication.id, cast(coalesce(abstract, true) as int) has_abstract from ${stats_db_name}.publication; ---ANALYZE TABLE ${stats_db_name}.indi_pub_has_abstract COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_result_with_orcid stored as parquet as select distinct r.id, coalesce(has_orcid, 0) as has_orcid from ${stats_db_name}.result r left outer join (select id, 1 as has_orcid from ${stats_db_name}.result_orcid) tmp on r.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_result_with_orcid COMPUTE STATISTICS; - ---- Sprint 3 ---- create table if not exists ${stats_db_name}.indi_funded_result_with_fundref stored as parquet as select distinct r.result as id, coalesce(fundref, 0) as fundref @@ -82,8 +68,6 @@ from ${stats_db_name}.project_results r where provenance='Harvested') tmp on r.result= tmp.result; ---ANALYZE TABLE ${stats_db_name}.indi_funded_result_with_fundref COMPUTE STATISTICS; - -- create table indi_result_org_collab stored as parquet as -- select o1.organization org1, o2.organization org2, count(distinct o1.id) as collaborations -- from result_organization as o1 @@ -103,8 +87,6 @@ group by o1.organization, o2.organization, o1.name, o2.name; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_result_org_collab COMPUTE STATISTICS; - create TEMPORARY TABLE ${stats_db_name}.tmp AS select distinct ro.organization organization, ro.id, o.name, o.country from ${stats_db_name}.result_organization ro join ${stats_db_name}.organization o on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; @@ -117,8 +99,6 @@ group by o1.organization, o1.id, o1.name, o2.country; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_result_org_country_collab COMPUTE STATISTICS; - create TEMPORARY TABLE ${stats_db_name}.tmp AS select o.id organization, o.name, ro.project as project from ${stats_db_name}.organization o join ${stats_db_name}.organization_projects ro on o.id=ro.id where o.name is not null; @@ -132,8 +112,6 @@ group by o1.name,o2.name, o1.organization, o2.organization; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_project_collab_org COMPUTE STATISTICS; - create TEMPORARY TABLE ${stats_db_name}.tmp AS select o.id organization, o.name, o.country , ro.project as project from ${stats_db_name}.organization o join ${stats_db_name}.organization_projects ro on o.id=ro.id @@ -148,8 +126,6 @@ group by o1.organization, o2.country, o1.name; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_project_collab_org_country COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_funder_country_collab stored as parquet as with tmp as (select funder, project, country from ${stats_db_name}.organization_projects op join ${stats_db_name}.organization o on o.id=op.id @@ -161,8 +137,6 @@ from tmp as f1 where f1.country<>f2.country group by f1.funder, f2.country, f1.country; ---ANALYZE TABLE ${stats_db_name}.indi_funder_country_collab COMPUTE STATISTICS; - create TEMPORARY TABLE ${stats_db_name}.tmp AS select distinct country, ro.id as result from ${stats_db_name}.organization o join ${stats_db_name}.result_organization ro on o.id=ro.organization @@ -177,8 +151,6 @@ group by o1.country, o2.country; drop table ${stats_db_name}.tmp purge; ---ANALYZE TABLE ${stats_db_name}.indi_result_country_collab COMPUTE STATISTICS; - ---- Sprint 4 ---- create table if not exists ${stats_db_name}.indi_pub_diamond stored as parquet as select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal @@ -190,8 +162,6 @@ from ${stats_db_name}.publication_datasources pd and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp on pd.id=tmp.id; -----ANALYZE TABLE ${stats_db_name}.indi_pub_diamond COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_in_transformative stored as parquet as select distinct pd.id, coalesce(is_transformative, 0) as is_transformative from ${stats_db_name}.publication pd @@ -202,8 +172,6 @@ from ${stats_db_name}.publication pd and ps.is_transformative_journal=true) tmp on pd.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_in_transformative COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_closed_other_open stored as parquet as select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from ${stats_db_name}.result_instance ri left outer join @@ -214,14 +182,10 @@ select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_op (p.bestlicence='Open Access' or p.bestlicence='Open Source')) tmp on tmp.id=ri.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_closed_other_open COMPUTE STATISTICS; - ---- Sprint 5 ---- create table if not exists ${stats_db_name}.indi_result_no_of_copies stored as parquet as select id, count(id) as number_of_copies from ${stats_db_name}.result_instance group by id; ---ANALYZE TABLE ${stats_db_name}.indi_result_no_of_copies COMPUTE STATISTICS; - ---- Sprint 6 ---- create table if not exists ${stats_db_name}.indi_pub_downloads stored as parquet as SELECT result_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats @@ -239,24 +203,18 @@ where downloads>0 GROUP BY result_id, repository_id order by result_id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_datasource COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_downloads_year stored as parquet as -SELECT result_id, substring(us.`date`, 1,4) as `year`, sum(downloads) no_downloads +SELECT result_id, cast(substring(us.`date`, 1,4) as int) as `year`, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, substring(us.`date`, 1,4); ---ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_year COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_downloads_datasource_year stored as parquet as -SELECT result_id, substring(us.`date`, 1,4) as `year`, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us +SELECT result_id, cast(substring(us.`date`, 1,4) as int) as `year`, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, repository_id, substring(us.`date`, 1,4); ---ANALYZE TABLE ${stats_db_name}.indi_pub_downloads_datasource_year COMPUTE STATISTICS; - ---- Sprint 7 ---- create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as WITH gold_oa AS ( SELECT @@ -307,8 +265,6 @@ FROM JOIN gold_oa on issn.issn = gold_oa.issn) tmp on pd.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_gold_oa COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc stored as parquet as WITH hybrid_oa AS ( SELECT issn_l, journal_is_in_doaj, journal_is_oa, issn_print as issn @@ -340,8 +296,6 @@ FROM ${stats_db_name}.publication_datasources pd JOIN ${stats_db_name}.indi_pub_gold_oa ga on pd.id=ga.id where cc.has_cc_license=1 and ga.is_gold=0) tmp on pd.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as WITH gold_oa AS ( SELECT issn_l, @@ -393,8 +347,6 @@ from ${stats_db_name}.publication_datasources pd where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp on pd.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_hybrid COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_org_fairness stored as parquet as --return results with PIDs, and rich metadata group by organization with result_fair as @@ -413,8 +365,6 @@ select allresults.organization, result_fair.no_result_fair/allresults.no_allresu from allresults join result_fair on result_fair.organization=allresults.organization; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness COMPUTE STATISTICS; - CREATE TEMPORARY table ${stats_db_name}.result_fair as select ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro @@ -439,8 +389,6 @@ from ${stats_db_name}.allresults ar DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub_pr COMPUTE STATISTICS; - CREATE TEMPORARY table ${stats_db_name}.result_fair as select year, ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro join ${stats_db_name}.result p on p.id=ro.id @@ -460,8 +408,6 @@ from ${stats_db_name}.allresults DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub_year COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.result_fair as select ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro @@ -484,8 +430,6 @@ on rf.organization=ar.organization; DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness_pub COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.result_fair as select year, ro.organization organization, count(distinct ro.id) no_result_fair from ${stats_db_name}.result_organization ro join ${stats_db_name}.result r on r.id=ro.id @@ -507,8 +451,6 @@ create table if not exists ${stats_db_name}.indi_org_fairness_year stored as par DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_fairness_year COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.result_with_pid as select year, ro.organization, count(distinct rp.id) no_result_with_pid from ${stats_db_name}.result_organization ro join ${stats_db_name}.result_pids rp on rp.id=ro.id @@ -530,8 +472,6 @@ from ${stats_db_name}.allresults DROP table ${stats_db_name}.result_with_pid purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_findable_year COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.result_with_pid as select ro.organization, count(distinct rp.id) no_result_with_pid from ${stats_db_name}.result_organization ro join ${stats_db_name}.result_pids rp on rp.id=ro.id @@ -553,8 +493,6 @@ from ${stats_db_name}.allresults DROP table ${stats_db_name}.result_with_pid purge; DROP table ${stats_db_name}.allresults purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_findable COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa as SELECT ro.organization, count(distinct r.id) no_oapubs FROM ${stats_db_name}.publication r join ${stats_db_name}.result_organization ro on ro.id=r.id @@ -633,8 +571,6 @@ DROP TABLE ${stats_db_name}.allpubsshare purge; DROP TABLE ${stats_db_name}.alldatasetssshare purge; DROP TABLE ${stats_db_name}.allsoftwaresshare purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_openess COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa AS SELECT r.year, ro.organization, count(distinct r.id) no_oapubs FROM ${stats_db_name}.publication r join ${stats_db_name}.result_organization ro on ro.id=r.id @@ -690,7 +626,7 @@ select allsoftware.year, software_oa.organization, software_oa.no_oasoftware/all create table if not exists ${stats_db_name}.indi_org_openess_year stored as parquet as -select allpubsshare.year, allpubsshare.organization, +select cast(allpubsshare.year as int), allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) +(case when d is null then 0 else 1 end)) org_openess FROM ${stats_db_name}.allpubsshare @@ -711,8 +647,6 @@ DROP TABLE ${stats_db_name}.allpubsshare purge; DROP TABLE ${stats_db_name}.alldatasetssshare purge; DROP TABLE ${stats_db_name}.allsoftwaresshare purge; ---ANALYZE TABLE ${stats_db_name}.indi_org_openess_year COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_has_preprint stored as parquet as select distinct p.id, coalesce(has_preprint, 0) as has_preprint from ${stats_db_name}.publication_classifications p @@ -722,8 +656,6 @@ from ${stats_db_name}.publication_classifications p where p.type='Preprint') tmp on p.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_has_preprint COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_in_subscribed stored as parquet as select distinct p.id, coalesce(is_subscription, 0) as is_subscription from ${stats_db_name}.publication p @@ -735,8 +667,6 @@ from ${stats_db_name}.publication p where g.is_gold=0 and h.is_hybrid=0 and t.is_transformative=0) tmp on p.id=tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_in_subscribed COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_result_with_pid as select distinct p.id, coalesce(result_with_pid, 0) as result_with_pid from ${stats_db_name}.result p @@ -745,8 +675,6 @@ from ${stats_db_name}.result p from ${stats_db_name}.result_pids p) tmp on p.id= tmp.id; ---ANALYZE TABLE ${stats_db_name}.indi_result_with_pid COMPUTE STATISTICS; - CREATE TEMPORARY TABLE ${stats_db_name}.pub_fos_totals as select rf.id, count(distinct lvl3) totals from ${stats_db_name}.result_fos rf group by rf.id; @@ -761,8 +689,6 @@ where totals>1) tmp on p.id=tmp.id; drop table ${stats_db_name}.pub_fos_totals purge; ---ANALYZE TABLE ${stats_db_name}.indi_pub_interdisciplinarity COMPUTE STATISTICS; - create table if not exists ${stats_db_name}.indi_pub_bronze_oa stored as parquet as select distinct p.id, coalesce(is_bronze_oa,0) as is_bronze_oa from ${stats_db_name}.publication p @@ -772,4 +698,280 @@ join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id where cc.has_cc_license=0 and ga.is_gold=0) tmp on tmp.id=p.id; ---ANALYZE TABLE ${stats_db_name}.indi_pub_bronze_oa COMPUTE STATISTICS; \ No newline at end of file +CREATE TEMPORARY TABLE ${stats_db_name}.project_year_result_year as +select p.id project_id, acronym, r.id result_id, r.year, p.end_year +from ${stats_db_name}.project p +join ${stats_db_name}.result_projects rp on p.id=rp.project +join ${stats_db_name}.result r on r.id=rp.id +where p.end_year is NOT NULL and r.year is not null; + +create table if not exists ${stats_db_name}.indi_is_project_result_after stored as parquet as +select pry.project_id, pry.acronym, pry.result_id, +coalesce(is_project_result_after, 0) as is_project_result_after +from ${stats_db_name}.project_year_result_year pry +left outer join (select pry.project_id, pry.acronym, pry.result_id, 1 as is_project_result_after +from ${stats_db_name}.project_year_result_year pry +where pry.year>pry.end_year) tmp on pry.result_id=tmp.result_id; + +drop table ${stats_db_name}.project_year_result_year purge; + +create table if not exists ${stats_db_name}.indi_is_funder_plan_s stored as parquet as +select distinct f.id, f.name, coalesce(is_funder_plan_s, 0) as is_funder_plan_s +from ${stats_db_name}.funder f + left outer join (select id, name, 1 as is_funder_plan_s from ${stats_db_name}.funder + join stats_ext.plan_s_short on c_o_alition_s_organisation_funder=name) tmp + on f.name= tmp.name; + +--Funder Fairness + +create table if not exists ${stats_db_name}.indi_funder_fairness stored as parquet as + with result_fair as + (select p.funder funder, count(distinct rp.id) no_result_fair from ${stats_db_name}.result_projects rp + join ${stats_db_name}.result r on r.id=rp.id + join ${stats_db_name}.project p on p.id=rp.project + where (r.title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 + group by p.funder), + allresults as (select p.funder funder, count(distinct rp.id) no_allresults from ${stats_db_name}.result_projects rp + join ${stats_db_name}.result r on r.id=rp.id + join ${stats_db_name}.project p on p.id=rp.project + where cast(year as int)>2003 + group by p.funder) +select allresults.funder, result_fair.no_result_fair/allresults.no_allresults funder_fairness +from allresults + join result_fair on result_fair.funder=allresults.funder; + +--RIs Fairness +create table if not exists ${stats_db_name}.indi_ris_fairness stored as parquet as +with result_contexts as +(select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc +join ${stats_db_name}.concept on concept.id=rc.concept +join ${stats_db_name}.category on category.id=concept.category +join ${stats_db_name}.context on context.id=category.context), +result_fair as + (select rc.ri_initiative ri_initiative, count(distinct rc.id) no_result_fair from result_contexts rc + join ${stats_db_name}.result r on r.id=rc.id + where (title is not null) and (publisher is not null) and (abstract=true) and (year is not null) and (authors>0) and cast(year as int)>2003 + group by rc.ri_initiative), +allresults as +(select rc.ri_initiative ri_initiative, count(distinct rc.id) no_allresults from result_contexts rc + join ${stats_db_name}.result r on r.id=rc.id + where cast(year as int)>2003 + group by rc.ri_initiative) +select allresults.ri_initiative, result_fair.no_result_fair/allresults.no_allresults ris_fairness +from allresults + join result_fair on result_fair.ri_initiative=allresults.ri_initiative; + +--Funder Openess + +CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa as +select p.funder funder, count(distinct rp.id) no_oapubs from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.publication r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by p.funder; + + +CREATE TEMPORARY TABLE ${stats_db_name}.datasets_oa as +select p.funder funder, count(distinct rp.id) no_oadatasets from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.dataset r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.software_oa as +select p.funder funder, count(distinct rp.id) no_oasoftware from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.software r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.allpubs as +select p.funder funder, count(distinct rp.id) no_allpubs from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.publication r on r.id=rp.id +where cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasets as +select p.funder funder, count(distinct rp.id) no_alldatasets from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.dataset r on r.id=rp.id +where cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftware as +select p.funder funder, count(distinct rp.id) no_allsoftware from ${stats_db_name}.result_projects rp +join ${stats_db_name}.project p on p.id=rp.project +join ${stats_db_name}.software r on r.id=rp.id +where cast(r.year as int)>2003 +group by p.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.allpubsshare as +select pubs_oa.funder, pubs_oa.no_oapubs/allpubs.no_allpubs p from ${stats_db_name}.allpubs + join ${stats_db_name}.pubs_oa on allpubs.funder=pubs_oa.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasetssshare as +select datasets_oa.funder, datasets_oa.no_oadatasets/alldatasets.no_alldatasets d + from ${stats_db_name}.alldatasets + join ${stats_db_name}.datasets_oa on alldatasets.funder=datasets_oa.funder; + +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftwaresshare as +select software_oa.funder, software_oa.no_oasoftware/allsoftware.no_allsoftware s + from ${stats_db_name}.allsoftware + join ${stats_db_name}.software_oa on allsoftware.funder=software_oa.funder; + +create table if not exists ${stats_db_name}.indi_funder_openess stored as parquet as +select allpubsshare.funder, + (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) + +(case when d is null then 0 else 1 end)) + funder_openess FROM ${stats_db_name}.allpubsshare + left outer join (select funder,d from + ${stats_db_name}.alldatasetssshare) tmp1 + on tmp1.funder=allpubsshare.funder + left outer join (select funder,s from + ${stats_db_name}.allsoftwaresshare) tmp2 + on tmp2.funder=allpubsshare.funder; + +DROP TABLE ${stats_db_name}.pubs_oa purge; +DROP TABLE ${stats_db_name}.datasets_oa purge; +DROP TABLE ${stats_db_name}.software_oa purge; +DROP TABLE ${stats_db_name}.allpubs purge; +DROP TABLE ${stats_db_name}.alldatasets purge; +DROP TABLE ${stats_db_name}.allsoftware purge; +DROP TABLE ${stats_db_name}.allpubsshare purge; +DROP TABLE ${stats_db_name}.alldatasetssshare purge; +DROP TABLE ${stats_db_name}.allsoftwaresshare purge; + +--RIs Openess + +CREATE TEMPORARY TABLE ${stats_db_name}.result_contexts as +select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc +join ${stats_db_name}.concept on concept.id=rc.concept +join ${stats_db_name}.category on category.id=concept.category +join ${stats_db_name}.context on context.id=category.context; + +CREATE TEMPORARY TABLE ${stats_db_name}.pubs_oa as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_oapubs from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.publication r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.datasets_oa as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_oadatasets from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.dataset r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.software_oa as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_oasoftware from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.software r on r.id=rp.id +join ${stats_db_name}.result_instance ri on ri.id=r.id +where (ri.accessright = 'Open Access' or ri.accessright = 'Embargo' or ri.accessright = 'Open Source') +and cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.allpubs as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_allpubs from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.publication r on r.id=rp.id +where cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasets as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_alldatasets from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.dataset r on r.id=rp.id +where cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftware as +select rp.ri_initiative ri_initiative, count(distinct rp.id) no_allsoftware from ${stats_db_name}.result_contexts rp +join ${stats_db_name}.software r on r.id=rp.id +where cast(r.year as int)>2003 +group by rp.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.allpubsshare as +select pubs_oa.ri_initiative, pubs_oa.no_oapubs/allpubs.no_allpubs p from ${stats_db_name}.allpubs + join ${stats_db_name}.pubs_oa on allpubs.ri_initiative=pubs_oa.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.alldatasetssshare as +select datasets_oa.ri_initiative, datasets_oa.no_oadatasets/alldatasets.no_alldatasets d + from ${stats_db_name}.alldatasets + join ${stats_db_name}.datasets_oa on alldatasets.ri_initiative=datasets_oa.ri_initiative; + +CREATE TEMPORARY TABLE ${stats_db_name}.allsoftwaresshare as +select software_oa.ri_initiative, software_oa.no_oasoftware/allsoftware.no_allsoftware s + from ${stats_db_name}.allsoftware + join ${stats_db_name}.software_oa on allsoftware.ri_initiative=software_oa.ri_initiative; + +create table if not exists ${stats_db_name}.indi_ris_openess stored as parquet as +select allpubsshare.ri_initiative, + (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) + +(case when d is null then 0 else 1 end)) + ris_openess FROM ${stats_db_name}.allpubsshare + left outer join (select ri_initiative,d from + ${stats_db_name}.alldatasetssshare) tmp1 + on tmp1.ri_initiative=allpubsshare.ri_initiative + left outer join (select ri_initiative,s from + ${stats_db_name}.allsoftwaresshare) tmp2 + on tmp2.ri_initiative=allpubsshare.ri_initiative; + +DROP TABLE ${stats_db_name}.result_contexts purge; +DROP TABLE ${stats_db_name}.pubs_oa purge; +DROP TABLE ${stats_db_name}.datasets_oa purge; +DROP TABLE ${stats_db_name}.software_oa purge; +DROP TABLE ${stats_db_name}.allpubs purge; +DROP TABLE ${stats_db_name}.alldatasets purge; +DROP TABLE ${stats_db_name}.allsoftware purge; +DROP TABLE ${stats_db_name}.allpubsshare purge; +DROP TABLE ${stats_db_name}.alldatasetssshare purge; +DROP TABLE ${stats_db_name}.allsoftwaresshare purge; + +--Funder Findability +create table if not exists ${stats_db_name}.indi_funder_findable stored as parquet as +with result_findable as + (select p.funder funder, count(distinct rp.id) no_result_findable from ${stats_db_name}.result_projects rp + join ${stats_db_name}.publication r on r.id=rp.id + join ${stats_db_name}.project p on p.id=rp.project + join ${stats_db_name}.result_pids rpi on rpi.id=r.id + where cast(year as int)>2003 + group by p.funder), + allresults as (select p.funder funder, count(distinct rp.id) no_allresults from ${stats_db_name}.result_projects rp + join ${stats_db_name}.result r on r.id=rp.id + join ${stats_db_name}.project p on p.id=rp.project + where cast(year as int)>2003 + group by p.funder) +select allresults.funder, result_findable.no_result_findable/allresults.no_allresults funder_findable +from allresults + join result_findable on result_findable.funder=allresults.funder; + +--RIs Findability +create table if not exists ${stats_db_name}.indi_ris_findable stored as parquet as +with result_contexts as +(select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc +join ${stats_db_name}.concept on concept.id=rc.concept +join ${stats_db_name}.category on category.id=concept.category +join ${stats_db_name}.context on context.id=category.context), +result_findable as + (select rc.ri_initiative ri_initiative, count(distinct rc.id) no_result_findable from result_contexts rc + join ${stats_db_name}.result r on r.id=rc.id + join ${stats_db_name}.result_pids rp on rp.id=r.id + where cast(r.year as int)>2003 + group by rc.ri_initiative), +allresults as +(select rc.ri_initiative ri_initiative, count(distinct rc.id) no_allresults from result_contexts rc + join ${stats_db_name}.result r on r.id=rc.id + where cast(r.year as int)>2003 + group by rc.ri_initiative) +select allresults.ri_initiative, result_findable.no_result_findable/allresults.no_allresults ris_findable +from allresults + join result_findable on result_findable.ri_initiative=allresults.ri_initiative; + diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index 3eeb792c7..586bee347 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -88,6 +88,7 @@ create view if not exists TARGET.doctoratestudents as select * from SOURCE.docto create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; +create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; @@ -270,3 +271,11 @@ create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * f --ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; +create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.result_id); +create table TARGET.indi_is_funder_plan_s stored as parquet as select * from SOURCE.indi_is_funder_plan_s orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create view TARGET.indi_funder_fairness as select * from SOURCE.indi_funder_fairness; +create view TARGET.indi_funder_openess as select * from SOURCE.indi_funder_openess; +create view TARGET.indi_funder_findable as select * from SOURCE.indi_funder_findable; +create view TARGET.indi_ris_fairness as select * from SOURCE.indi_ris_fairness; +create view TARGET.indi_ris_openess as select * from SOURCE.indi_ris_openess; +create view TARGET.indi_ris_findable as select * from SOURCE.indi_ris_findable; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql index a59791084..df4795e3e 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDBAll.sql @@ -15,6 +15,7 @@ create view if not exists TARGET.doctoratestudents as select * from SOURCE.docto create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; +create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; create table TARGET.result stored as parquet as select distinct * from ( @@ -73,7 +74,11 @@ create table TARGET.result stored as parquet as 'openorgs____::c0b262bd6eab819e4c994914f9c010e2', -- National Institute of Geophysics and Volcanology 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa - 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab', -- Universidade Católica Portuguesa + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', -- Iscte - Instituto Universitário de Lisboa + 'openorgs____::5d55fb216b14691cf68218daf5d78cd9', -- Munster Technological University + 'openorgs____::0fccc7640f0cb44d5cd1b06b312a06b9', -- Cardiff University + 'openorgs____::8839b55dae0c84d56fd533f52d5d483a' -- Leibniz Institute of Ecological Urban and Regional Development ) )) foo; --ANALYZE TABLE TARGET.result COMPUTE STATISTICS; @@ -92,6 +97,7 @@ create view if not exists TARGET.doctoratestudents as select * from SOURCE.docto create view if not exists TARGET.totalresearchers as select * from SOURCE.totalresearchers; create view if not exists TARGET.totalresearchersft as select * from SOURCE.totalresearchersft; create view if not exists TARGET.hrrst as select * from SOURCE.hrrst; +--create view if not exists TARGET.graduatedoctorates as select * from SOURCE.graduatedoctorates; create table TARGET.result_citations stored as parquet as select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; @@ -274,3 +280,13 @@ create table TARGET.indi_pub_interdisciplinarity stored as parquet as select * f --ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; create table TARGET.result_apc_affiliations stored as parquet as select * from SOURCE.result_apc_affiliations orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_apc_affiliations COMPUTE STATISTICS; +create table TARGET.indi_is_project_result_after stored as parquet as select * from SOURCE.indi_is_project_result_after orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create table TARGET.indi_is_funder_plan_s stored as parquet as select * from SOURCE.indi_is_funder_plan_s orig where exists (select 1 from TARGET.result r where r.id=orig.id); +create view TARGET.indi_funder_fairness as select * from SOURCE.indi_funder_fairness; +create view TARGET.indi_funder_openess as select * from SOURCE.indi_funder_openess; +create view TARGET.indi_funder_findable as select * from SOURCE.indi_funder_findable; +create view TARGET.indi_ris_fairness as select * from SOURCE.indi_ris_fairness; +create view TARGET.indi_ris_openess as select * from SOURCE.indi_ris_openess; +create view TARGET.indi_ris_findable as select * from SOURCE.indi_ris_findable; + + diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql index 1f75c3cd1..7bfba92a8 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB_institutions.sql @@ -34,16 +34,16 @@ create table TARGET.result stored as parquet as 'openorgs____::3e8d1f8c3f6cd7f418b09f1f58b4873b', -- Aristotle University of Thessaloniki 'openorgs____::3fcef6e1c469c10f2a84b281372c9814', -- World Bank 'openorgs____::1698a2eb1885ef8adb5a4a969e745ad3', -- École des Ponts ParisTech - 'openorgs____::e15adb13c4dadd49de4d35c39b5da93a', -- Nanyang Technological University - 'openorgs____::4b34103bde246228fcd837f5f1bf4212', -- Autonomous University of Barcelona - 'openorgs____::72ec75fcfc4e0df1a76dc4c49007fceb', -- McMaster University - 'openorgs____::51c7fc556e46381734a25a6fbc3fd398', -- University of Modena and Reggio Emilia - 'openorgs____::235d7f9ad18ecd7e6dc62ea4990cb9db', -- Bilkent University - 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje + 'openorgs____::e15adb13c4dadd49de4d35c39b5da93a', -- Nanyang Technological University + 'openorgs____::4b34103bde246228fcd837f5f1bf4212', -- Autonomous University of Barcelona + 'openorgs____::72ec75fcfc4e0df1a76dc4c49007fceb', -- McMaster University + 'openorgs____::51c7fc556e46381734a25a6fbc3fd398', -- University of Modena and Reggio Emilia + 'openorgs____::235d7f9ad18ecd7e6dc62ea4990cb9db', -- Bilkent University + 'openorgs____::31f2fa9e05b49d4cf40a19c3fed8eb06', -- Saints Cyril and Methodius University of Skopje 'openorgs____::db7686f30f22cbe73a4fde872ce812a6', -- University of Milan - 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork + 'openorgs____::b8b8ca674452579f3f593d9f5e557483', -- University College Cork 'openorgs____::38d7097854736583dde879d12dacafca', -- Brown University - 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech + 'openorgs____::57784c9e047e826fefdb1ef816120d92', --Arts et Métiers ParisTech 'openorgs____::2530baca8a15936ba2e3297f2bce2e7e', -- University of Cape Town 'openorgs____::d11f981828c485cd23d93f7f24f24db1', -- Technological University Dublin 'openorgs____::5e6bf8962665cdd040341171e5c631d8', -- Delft University of Technology @@ -52,10 +52,13 @@ create table TARGET.result stored as parquet as 'openorgs____::66aa9fc2fceb271423dfabcc38752dc0', -- Lund University 'openorgs____::3cff625a4370d51e08624cc586138b2f', -- IMT Atlantique 'openorgs____::c0b262bd6eab819e4c994914f9c010e2', -- National Institute of Geophysics and Volcanology - 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam + 'openorgs____::1624ff7c01bb641b91f4518539a0c28a', -- Vrije Universiteit Amsterdam 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa - 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa - + 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab', -- Universidade Católica Portuguesa + 'openorgs____::4d4051b56708688235252f1d8fddb8c1', -- Iscte - Instituto Universitário de Lisboa + 'openorgs____::5d55fb216b14691cf68218daf5d78cd9', -- Munster Technological University + 'openorgs____::0fccc7640f0cb44d5cd1b06b312a06b9', -- Cardiff University + 'openorgs____::8839b55dae0c84d56fd533f52d5d483a' -- Leibniz Institute of Ecological Urban and Regional Development ))) foo; --ANALYZE TABLE TARGET.result COMPUTE STATISTICS; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql old mode 100644 new mode 100755 diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml index c03520e48..aa991730b 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml @@ -467,7 +467,7 @@ ${usage_stats_db_shadow_name} finalizeImpalaCluster.sh - + From 5f90cc11e98d0addbfb22bf8ce0a83e87a269e00 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Wed, 6 Sep 2023 14:14:38 +0300 Subject: [PATCH 04/12] Update step16-createIndicatorsTables.sql Fix indi_pub_bronze_oa --- .../oozie_app/scripts/step16-createIndicatorsTables.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index 1c80f6757..dd249d371 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -696,7 +696,11 @@ left outer join (select p.id, 1 as is_bronze_oa from ${stats_db_name}.publication p join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id -where cc.has_cc_license=0 and ga.is_gold=0) tmp on tmp.id=p.id; +join ${stats_db_name}.result_instance ri on ri.id=p.id +join ${stats_db_name}.datasource d on d.id=ri.hostedby +where cc.has_cc_license=0 and ga.is_gold=0 +and (d.type='Journal' or d.type='Journal Aggregator/Publisher') +and ri.accessright='Open Access') tmp on tmp.id=p.id; CREATE TEMPORARY TABLE ${stats_db_name}.project_year_result_year as select p.id project_id, acronym, r.id result_id, r.year, p.end_year From 9ef971a1464e5d307c407316cda69eb97d6ecb9a Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Tue, 19 Sep 2023 14:25:42 +0300 Subject: [PATCH 05/12] Update step16-createIndicatorsTables.sql Fix int year for: indi_org_openess_year indi_org_fairness_year indi_org_findable_year --- .../scripts/step16-createIndicatorsTables.sql | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index dd249d371..ae95727a6 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -444,9 +444,9 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as group by ro.organization, year; create table if not exists ${stats_db_name}.indi_org_fairness_year stored as parquet as - select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness + select cast(allresults.year as int) year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from ${stats_db_name}.allresults - join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and result_fair.year=allresults.year; + join ${stats_db_name}.result_fair on result_fair.organization=allresults.organization and cast(result_fair.year as int)=cast(allresults.year as int); DROP table ${stats_db_name}.result_fair purge; DROP table ${stats_db_name}.allresults purge; @@ -465,9 +465,9 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as group by ro.organization, year; create table if not exists ${stats_db_name}.indi_org_findable_year stored as parquet as -select allresults.year, allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable +select cast(allresults.year as int) year, allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable from ${stats_db_name}.allresults - join ${stats_db_name}.result_with_pid on result_with_pid.organization=allresults.organization and result_with_pid.year=allresults.year; + join ${stats_db_name}.result_with_pid on result_with_pid.organization=allresults.organization and cast(result_with_pid.year as int)=cast(allresults.year as int); DROP table ${stats_db_name}.result_with_pid purge; DROP table ${stats_db_name}.allresults purge; @@ -626,16 +626,16 @@ select allsoftware.year, software_oa.organization, software_oa.no_oasoftware/all create table if not exists ${stats_db_name}.indi_org_openess_year stored as parquet as -select cast(allpubsshare.year as int), allpubsshare.organization, +select cast(allpubsshare.year as int) year, allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) +(case when d is null then 0 else 1 end)) org_openess FROM ${stats_db_name}.allpubsshare - left outer join (select year, organization,d from + left outer join (select cast(year as int), organization,d from ${stats_db_name}.alldatasetssshare) tmp1 on tmp1.organization=allpubsshare.organization and tmp1.year=allpubsshare.year - left outer join (select year, organization,s from + left outer join (select cast(year as int), organization,s from ${stats_db_name}.allsoftwaresshare) tmp2 - on tmp2.organization=allpubsshare.organization and tmp2.year=allpubsshare.year; + on tmp2.organization=allpubsshare.organization and cast(tmp2.year as int)=cast(allpubsshare.year as int); DROP TABLE ${stats_db_name}.pubs_oa purge; DROP TABLE ${stats_db_name}.datasets_oa purge; From 489a082f044cc89215f2183eb06ff764826f8578 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Mon, 9 Oct 2023 14:00:50 +0300 Subject: [PATCH 06/12] Update step16-createIndicatorsTables.sql Change scripts for gold, hybrid, bronze indicators --- .../scripts/step16-createIndicatorsTables.sql | 353 ++++++++++++------ 1 file changed, 245 insertions(+), 108 deletions(-) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql index ae95727a6..6af486340 100755 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql @@ -1,4 +1,6 @@ -- Sprint 1 ---- +drop table if exists ${stats_db_name}.indi_pub_green_oa purge; + create table if not exists ${stats_db_name}.indi_pub_green_oa stored as parquet as select distinct p.id, coalesce(green_oa, 0) as green_oa from ${stats_db_name}.publication p @@ -12,6 +14,8 @@ from ${stats_db_name}.publication p or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp on p.id= tmp.id; +drop table if exists ${stats_db_name}.indi_pub_grey_lit purge; + create table if not exists ${stats_db_name}.indi_pub_grey_lit stored as parquet as select distinct p.id, coalesce(grey_lit, 0) as grey_lit from ${stats_db_name}.publication p @@ -23,6 +27,8 @@ from ${stats_db_name}.publication p not exists (select 1 from ${stats_db_name}.result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id; +drop table if exists ${stats_db_name}.indi_pub_doi_from_crossref purge; + create table if not exists ${stats_db_name}.indi_pub_doi_from_crossref stored as parquet as select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref from ${stats_db_name}.publication p @@ -33,6 +39,8 @@ from ${stats_db_name}.publication p on tmp.id=p.id; -- Sprint 2 ---- +drop table if exists ${stats_db_name}.indi_result_has_cc_licence purge; + create table if not exists ${stats_db_name}.indi_result_has_cc_licence stored as parquet as select distinct r.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license from ${stats_db_name}.result r @@ -41,6 +49,8 @@ left outer join (select r.id, license.type as lic from ${stats_db_name}.result r where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp on r.id= tmp.id; +drop table if exists ${stats_db_name}.indi_result_has_cc_licence_url purge; + create table if not exists ${stats_db_name}.indi_result_has_cc_licence_url stored as parquet as select distinct r.id, case when lic_host='' or lic_host is null then 0 else 1 end as has_cc_license_url from ${stats_db_name}.result r @@ -50,10 +60,14 @@ from ${stats_db_name}.result r WHERE lower(parse_url(license.type, "HOST")) = "creativecommons.org") tmp on r.id= tmp.id; +drop table if exists ${stats_db_name}.indi_pub_has_abstract purge; + create table if not exists ${stats_db_name}.indi_pub_has_abstract stored as parquet as select distinct publication.id, cast(coalesce(abstract, true) as int) has_abstract from ${stats_db_name}.publication; +drop table if exists ${stats_db_name}.indi_result_with_orcid purge; + create table if not exists ${stats_db_name}.indi_result_with_orcid stored as parquet as select distinct r.id, coalesce(has_orcid, 0) as has_orcid from ${stats_db_name}.result r @@ -61,6 +75,9 @@ from ${stats_db_name}.result r on r.id= tmp.id; ---- Sprint 3 ---- + +drop table if exists ${stats_db_name}.indi_funded_result_with_fundref purge; + create table if not exists ${stats_db_name}.indi_funded_result_with_fundref stored as parquet as select distinct r.result as id, coalesce(fundref, 0) as fundref from ${stats_db_name}.project_results r @@ -79,6 +96,8 @@ from ${stats_db_name}.project_results r create TEMPORARY TABLE ${stats_db_name}.tmp AS SELECT ro.organization organization, ro.id, o.name from ${stats_db_name}.result_organization ro join ${stats_db_name}.organization o on o.id=ro.organization where o.name is not null; +drop table if exists ${stats_db_name}.indi_result_org_collab purge; + create table if not exists ${stats_db_name}.indi_result_org_collab stored as parquet as select o1.organization org1, o1.name org1name1, o2.organization org2, o2.name org2name2, count(o1.id) as collaborations from ${stats_db_name}.tmp as o1 @@ -91,6 +110,8 @@ create TEMPORARY TABLE ${stats_db_name}.tmp AS select distinct ro.organization organization, ro.id, o.name, o.country from ${stats_db_name}.result_organization ro join ${stats_db_name}.organization o on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; +drop table if exists ${stats_db_name}.indi_result_org_country_collab purge; + create table if not exists ${stats_db_name}.indi_result_org_country_collab stored as parquet as select o1.organization org1,o1.name org1name1, o2.country country2, count(o1.id) as collaborations from ${stats_db_name}.tmp as o1 join ${stats_db_name}.tmp as o2 on o1.id=o2.id @@ -103,6 +124,8 @@ create TEMPORARY TABLE ${stats_db_name}.tmp AS select o.id organization, o.name, ro.project as project from ${stats_db_name}.organization o join ${stats_db_name}.organization_projects ro on o.id=ro.id where o.name is not null; +drop table if exists ${stats_db_name}.indi_project_collab_org purge; + create table if not exists ${stats_db_name}.indi_project_collab_org stored as parquet as select o1.organization org1,o1.name orgname1, o2.organization org2, o2.name orgname2, count(distinct o1.project) as collaborations from ${stats_db_name}.tmp as o1 @@ -117,6 +140,8 @@ select o.id organization, o.name, o.country , ro.project as project from ${stat join ${stats_db_name}.organization_projects ro on o.id=ro.id and o.country <> 'UNKNOWN' and o.name is not null; +drop table if exists ${stats_db_name}.indi_project_collab_org_country purge; + create table if not exists ${stats_db_name}.indi_project_collab_org_country stored as parquet as select o1.organization org1,o1.name org1name, o2.country country2, count(distinct o1.project) as collaborations from ${stats_db_name}.tmp as o1 @@ -126,6 +151,8 @@ group by o1.organization, o2.country, o1.name; drop table ${stats_db_name}.tmp purge; +drop table if exists ${stats_db_name}.indi_funder_country_collab purge; + create table if not exists ${stats_db_name}.indi_funder_country_collab stored as parquet as with tmp as (select funder, project, country from ${stats_db_name}.organization_projects op join ${stats_db_name}.organization o on o.id=op.id @@ -142,6 +169,8 @@ select distinct country, ro.id as result from ${stats_db_name}.organization o join ${stats_db_name}.result_organization ro on o.id=ro.organization where country <> 'UNKNOWN' and o.name is not null; +drop table if exists ${stats_db_name}.indi_result_country_collab purge; + create table if not exists ${stats_db_name}.indi_result_country_collab stored as parquet as select o1.country country1, o2.country country2, count(o1.result) as collaborations from ${stats_db_name}.tmp as o1 @@ -152,6 +181,8 @@ group by o1.country, o2.country; drop table ${stats_db_name}.tmp purge; ---- Sprint 4 ---- +drop table if exists ${stats_db_name}.indi_pub_diamond purge; + create table if not exists ${stats_db_name}.indi_pub_diamond stored as parquet as select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal from ${stats_db_name}.publication_datasources pd @@ -162,6 +193,8 @@ from ${stats_db_name}.publication_datasources pd and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp on pd.id=tmp.id; +drop table if exists ${stats_db_name}.indi_pub_in_transformative purge; + create table if not exists ${stats_db_name}.indi_pub_in_transformative stored as parquet as select distinct pd.id, coalesce(is_transformative, 0) as is_transformative from ${stats_db_name}.publication pd @@ -172,6 +205,8 @@ from ${stats_db_name}.publication pd and ps.is_transformative_journal=true) tmp on pd.id=tmp.id; +drop table if exists ${stats_db_name}.indi_pub_closed_other_open purge; + create table if not exists ${stats_db_name}.indi_pub_closed_other_open stored as parquet as select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from ${stats_db_name}.result_instance ri left outer join @@ -183,10 +218,14 @@ select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_op on tmp.id=ri.id; ---- Sprint 5 ---- +drop table if exists ${stats_db_name}.indi_result_no_of_copies purge; + create table if not exists ${stats_db_name}.indi_result_no_of_copies stored as parquet as select id, count(id) as number_of_copies from ${stats_db_name}.result_instance group by id; ---- Sprint 6 ---- +drop table if exists ${stats_db_name}.indi_pub_downloads purge; + create table if not exists ${stats_db_name}.indi_pub_downloads stored as parquet as SELECT result_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats join ${stats_db_name}.publication on result_id=id @@ -196,6 +235,8 @@ order by no_downloads desc; --ANALYZE TABLE ${stats_db_name}.indi_pub_downloads COMPUTE STATISTICS; +drop table if exists ${stats_db_name}.indi_pub_downloads_datasource purge; + create table if not exists ${stats_db_name}.indi_pub_downloads_datasource stored as parquet as SELECT result_id, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats join ${stats_db_name}.publication on result_id=id @@ -203,12 +244,16 @@ where downloads>0 GROUP BY result_id, repository_id order by result_id; +drop table if exists ${stats_db_name}.indi_pub_downloads_year purge; + create table if not exists ${stats_db_name}.indi_pub_downloads_year stored as parquet as SELECT result_id, cast(substring(us.`date`, 1,4) as int) as `year`, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us join ${stats_db_name}.publication on result_id=id where downloads>0 GROUP BY result_id, substring(us.`date`, 1,4); +drop table if exists ${stats_db_name}.indi_pub_downloads_datasource_year purge; + create table if not exists ${stats_db_name}.indi_pub_downloads_datasource_year stored as parquet as SELECT result_id, cast(substring(us.`date`, 1,4) as int) as `year`, repository_id, sum(downloads) no_downloads from openaire_prod_usage_stats.usage_stats us join ${stats_db_name}.publication on result_id=id @@ -216,54 +261,81 @@ where downloads>0 GROUP BY result_id, repository_id, substring(us.`date`, 1,4); ---- Sprint 7 ---- +drop table if exists ${stats_db_name}.indi_pub_gold_oa purge; + +--create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as +-- WITH gold_oa AS ( SELECT +-- issn_l, +-- journal_is_in_doaj, +-- journal_is_oa, +-- issn_1 as issn +-- FROM +-- STATS_EXT.oa_journals +-- WHERE +-- issn_1 != "" +-- UNION +-- ALL SELECT +-- issn_l, +-- journal_is_in_doaj, +-- journal_is_oa, +-- issn_2 as issn +-- FROM +-- STATS_EXT.oa_journals +-- WHERE +-- issn_2 != "" ), issn AS ( SELECT +-- * +-- FROM +--( SELECT +-- id, +-- issn_printed as issn +-- FROM +-- ${stats_db_name}.datasource +-- WHERE +-- issn_printed IS NOT NULL +-- UNION ALL +-- SELECT +-- id, +-- issn_online as issn +-- FROM +-- ${stats_db_name}.datasource +-- WHERE +-- issn_online IS NOT NULL or id like '%doajarticles%') as issn +-- WHERE +-- LENGTH(issn) > 7) +--SELECT +-- DISTINCT pd.id, coalesce(is_gold, 0) as is_gold +--FROM +-- ${stats_db_name}.publication_datasources pd +-- left outer join( +-- select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd +-- JOIN issn on issn.id=pd.datasource +-- JOIN gold_oa on issn.issn = gold_oa.issn) tmp +-- on pd.id=tmp.id; + create table if not exists ${stats_db_name}.indi_pub_gold_oa stored as parquet as - WITH gold_oa AS ( SELECT - issn_l, - journal_is_in_doaj, - journal_is_oa, - issn_1 as issn - FROM - STATS_EXT.oa_journals - WHERE - issn_1 != "" - UNION - ALL SELECT - issn_l, - journal_is_in_doaj, - journal_is_oa, - issn_2 as issn - FROM - STATS_EXT.oa_journals - WHERE - issn_2 != "" ), issn AS ( SELECT - * - FROM -( SELECT - id, - issn_printed as issn - FROM - ${stats_db_name}.datasource - WHERE - issn_printed IS NOT NULL - UNION ALL - SELECT - id, - issn_online as issn - FROM - ${stats_db_name}.datasource - WHERE - issn_online IS NOT NULL or id like '%doajarticles%') as issn - WHERE - LENGTH(issn) > 7) -SELECT - DISTINCT pd.id, coalesce(is_gold, 0) as is_gold -FROM - ${stats_db_name}.publication_datasources pd - left outer join( - select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd - JOIN issn on issn.id=pd.datasource - JOIN gold_oa on issn.issn = gold_oa.issn) tmp - on pd.id=tmp.id; +with gold_oa as ( +SELECT issn,issn_l from stats_ext.issn_gold_oa_dataset_v5), +issn AS (SELECT * FROM +(SELECT id,issn_printed as issn FROM ${stats_db_name}.datasource +WHERE issn_printed IS NOT NULL +UNION ALL +SELECT id, issn_online as issn FROM ${stats_db_name}.datasource +WHERE issn_online IS NOT NULL or id like '%doajarticles%') as issn +WHERE LENGTH(issn) > 7), +alljournals AS(select issn, issn_l from stats_ext.alljournals +where journal_is_in_doaj=true or journal_is_oa=true) +SELECT DISTINCT pd.id, coalesce(is_gold, 0) as is_gold +FROM ${stats_db_name}.publication_datasources pd +left outer join ( +select pd.id, 1 as is_gold FROM ${stats_db_name}.publication_datasources pd +JOIN issn on issn.id=pd.datasource +JOIN gold_oa on issn.issn = gold_oa.issn +join alljournals on issn.issn=alljournals.issn +left outer join ${stats_db_name}.result_instance ri on ri.id=pd.id +and ri.accessright!='Closed Access' and ri.accessright_uw='gold') tmp +on pd.id=tmp.id; + +drop table if exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc purge; create table if not exists ${stats_db_name}.indi_pub_hybrid_oa_with_cc stored as parquet as WITH hybrid_oa AS ( @@ -296,56 +368,71 @@ FROM ${stats_db_name}.publication_datasources pd JOIN ${stats_db_name}.indi_pub_gold_oa ga on pd.id=ga.id where cc.has_cc_license=1 and ga.is_gold=0) tmp on pd.id=tmp.id; +drop table if exists ${stats_db_name}.indi_pub_hybrid purge; + +--create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as +-- WITH gold_oa AS ( SELECT +-- issn_l, +-- journal_is_in_doaj, +-- journal_is_oa, +-- issn_1 as issn, +-- has_apc +-- FROM +-- STATS_EXT.oa_journals +-- WHERE +-- issn_1 != "" +-- UNION +-- ALL SELECT +-- issn_l, +-- journal_is_in_doaj, +-- journal_is_oa, +-- issn_2 as issn, +-- has_apc +-- FROM +-- STATS_EXT.oa_journals +-- WHERE +-- issn_2 != "" ), issn AS ( SELECT +-- * +-- FROM +--( SELECT +-- id, +-- issn_printed as issn +-- FROM +-- ${stats_db_name}.datasource +-- WHERE +-- issn_printed IS NOT NULL +-- UNION ALL +-- SELECT +-- id, +-- issn_online as issn +-- FROM +-- ${stats_db_name}.datasource +-- WHERE +-- issn_online IS NOT NULL or id like '%doajarticles%') as issn +-- WHERE +-- LENGTH(issn) > 7) +--select distinct pd.id, coalesce(is_hybrid, 0) as is_hybrid +--from ${stats_db_name}.publication_datasources pd +-- left outer join ( +-- select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd +-- join ${stats_db_name}.datasource d on d.id=pd.datasource +-- join issn on issn.id=pd.datasource +-- join gold_oa on issn.issn=gold_oa.issn +-- where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp +-- on pd.id=tmp.id; + create table if not exists ${stats_db_name}.indi_pub_hybrid stored as parquet as - WITH gold_oa AS ( SELECT - issn_l, - journal_is_in_doaj, - journal_is_oa, - issn_1 as issn, - has_apc - FROM - STATS_EXT.oa_journals - WHERE - issn_1 != "" - UNION - ALL SELECT - issn_l, - journal_is_in_doaj, - journal_is_oa, - issn_2 as issn, - has_apc - FROM - STATS_EXT.oa_journals - WHERE - issn_2 != "" ), issn AS ( SELECT - * - FROM -( SELECT - id, - issn_printed as issn - FROM - ${stats_db_name}.datasource - WHERE - issn_printed IS NOT NULL - UNION ALL - SELECT - id, - issn_online as issn - FROM - ${stats_db_name}.datasource - WHERE - issn_online IS NOT NULL or id like '%doajarticles%') as issn - WHERE - LENGTH(issn) > 7) -select distinct pd.id, coalesce(is_hybrid, 0) as is_hybrid -from ${stats_db_name}.publication_datasources pd - left outer join ( - select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd - join ${stats_db_name}.datasource d on d.id=pd.datasource - join issn on issn.id=pd.datasource - join gold_oa on issn.issn=gold_oa.issn - where (gold_oa.journal_is_in_doaj=false or gold_oa.journal_is_oa=false))tmp - on pd.id=tmp.id; +select pd.id,coalesce(is_hybrid,0) is_hybrid from ${stats_db_name}.publication_datasources pd +left outer join (select pd.id, 1 as is_hybrid from ${stats_db_name}.publication_datasources pd +join ${stats_db_name}.datasource d on pd.datasource=d.id +join ${stats_db_name}.result_instance ri on ri.id=pd.id +join ${stats_db_name}.indi_pub_gold_oa indi_gold on indi_gold.id=pd.id +join ${stats_db_name}.result_accessroute ra on ra.id=pd.id +where d.type like '%Journal%' and ri.accessright!='Closed Access' and (ri.accessright_uw!='gold' +or indi_gold.is_gold=0) and (ra.accessroute='hybrid' or ri.license is not null)) tmp +on pd.id=tmp.id; + +drop table if exists ${stats_db_name}.indi_org_fairness purge; create table if not exists ${stats_db_name}.indi_org_fairness stored as parquet as --return results with PIDs, and rich metadata group by organization @@ -381,6 +468,8 @@ select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name where cast(year as int)>2003 group by ro.organization; +drop table if exists ${stats_db_name}.indi_org_fairness_pub_pr purge; + create table if not exists ${stats_db_name}.indi_org_fairness_pub_pr stored as parquet as select ar.organization, rf.no_result_fair/ar.no_allresults org_fairness from ${stats_db_name}.allresults ar @@ -400,6 +489,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as select year, ro.organizati where cast(year as int)>2003 group by ro.organization, year; +drop table if exists ${stats_db_name}.indi_org_fairness_pub_year purge; + create table if not exists ${stats_db_name}.indi_org_fairness_pub_year stored as parquet as select allresults.year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from ${stats_db_name}.allresults @@ -422,6 +513,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as where cast(year as int)>2003 group by ro.organization; +drop table if exists ${stats_db_name}.indi_org_fairness_pub purge; + create table if not exists ${stats_db_name}.indi_org_fairness_pub as select ar.organization, rf.no_result_fair/ar.no_allresults org_fairness from ${stats_db_name}.allresults ar join ${stats_db_name}.result_fair rf @@ -443,6 +536,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as where cast(year as int)>2003 group by ro.organization, year; +drop table if exists ${stats_db_name}.indi_org_fairness_year purge; + create table if not exists ${stats_db_name}.indi_org_fairness_year stored as parquet as select cast(allresults.year as int) year, allresults.organization, result_fair.no_result_fair/allresults.no_allresults org_fairness from ${stats_db_name}.allresults @@ -464,6 +559,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.allresults as where cast(year as int) >2003 group by ro.organization, year; +drop table if exists ${stats_db_name}.indi_org_findable_year purge; + create table if not exists ${stats_db_name}.indi_org_findable_year stored as parquet as select cast(allresults.year as int) year, allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable from ${stats_db_name}.allresults @@ -485,6 +582,8 @@ select ro.organization, count(distinct ro.id) no_allresults from ${stats_db_name where cast(year as int) >2003 group by ro.organization; +drop table if exists ${stats_db_name}.indi_org_findable purge; + create table if not exists ${stats_db_name}.indi_org_findable stored as parquet as select allresults.organization, result_with_pid.no_result_with_pid/allresults.no_allresults org_findable from ${stats_db_name}.allresults @@ -549,6 +648,8 @@ select software_oa.organization, software_oa.no_oasoftware/allsoftware.no_allsof from ${stats_db_name}.allsoftware join ${stats_db_name}.software_oa on allsoftware.organization=software_oa.organization; +drop table if exists ${stats_db_name}.indi_org_openess purge; + create table if not exists ${stats_db_name}.indi_org_openess stored as parquet as select allpubsshare.organization, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) @@ -624,6 +725,7 @@ select allsoftware.year, software_oa.organization, software_oa.no_oasoftware/all from ${stats_db_name}.allsoftware join ${stats_db_name}.software_oa on allsoftware.organization=software_oa.organization where cast(allsoftware.year as INT)=cast(software_oa.year as int); +drop table if exists ${stats_db_name}.indi_org_openess_year purge; create table if not exists ${stats_db_name}.indi_org_openess_year stored as parquet as select cast(allpubsshare.year as int) year, allpubsshare.organization, @@ -647,6 +749,8 @@ DROP TABLE ${stats_db_name}.allpubsshare purge; DROP TABLE ${stats_db_name}.alldatasetssshare purge; DROP TABLE ${stats_db_name}.allsoftwaresshare purge; +drop table if exists ${stats_db_name}.indi_pub_has_preprint purge; + create table if not exists ${stats_db_name}.indi_pub_has_preprint stored as parquet as select distinct p.id, coalesce(has_preprint, 0) as has_preprint from ${stats_db_name}.publication_classifications p @@ -655,6 +759,7 @@ from ${stats_db_name}.publication_classifications p from ${stats_db_name}.publication_classifications p where p.type='Preprint') tmp on p.id= tmp.id; +drop table if exists ${stats_db_name}.indi_pub_in_subscribed purge; create table if not exists ${stats_db_name}.indi_pub_in_subscribed stored as parquet as select distinct p.id, coalesce(is_subscription, 0) as is_subscription @@ -667,6 +772,8 @@ from ${stats_db_name}.publication p where g.is_gold=0 and h.is_hybrid=0 and t.is_transformative=0) tmp on p.id=tmp.id; +drop table if exists ${stats_db_name}.indi_result_with_pid purge; + create table if not exists ${stats_db_name}.indi_result_with_pid as select distinct p.id, coalesce(result_with_pid, 0) as result_with_pid from ${stats_db_name}.result p @@ -679,6 +786,8 @@ CREATE TEMPORARY TABLE ${stats_db_name}.pub_fos_totals as select rf.id, count(distinct lvl3) totals from ${stats_db_name}.result_fos rf group by rf.id; +drop table if exists ${stats_db_name}.indi_pub_interdisciplinarity purge; + create table if not exists ${stats_db_name}.indi_pub_interdisciplinarity as select distinct p.id as id, coalesce(is_interdisciplinary, 0) as is_interdisciplinary @@ -689,18 +798,31 @@ where totals>1) tmp on p.id=tmp.id; drop table ${stats_db_name}.pub_fos_totals purge; -create table if not exists ${stats_db_name}.indi_pub_bronze_oa stored as parquet as -select distinct p.id, coalesce(is_bronze_oa,0) as is_bronze_oa -from ${stats_db_name}.publication p -left outer join -(select p.id, 1 as is_bronze_oa from ${stats_db_name}.publication p -join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id -join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id -join ${stats_db_name}.result_instance ri on ri.id=p.id -join ${stats_db_name}.datasource d on d.id=ri.hostedby -where cc.has_cc_license=0 and ga.is_gold=0 -and (d.type='Journal' or d.type='Journal Aggregator/Publisher') -and ri.accessright='Open Access') tmp on tmp.id=p.id; +drop table if exists ${stats_db_name}.indi_pub_bronze_oa purge; + +--create table if not exists ${stats_db_name}.indi_pub_bronze_oa stored as parquet as +--select distinct p.id, coalesce(is_bronze_oa,0) as is_bronze_oa +--from ${stats_db_name}.publication p +--left outer join +--(select p.id, 1 as is_bronze_oa from ${stats_db_name}.publication p +--join ${stats_db_name}.indi_result_has_cc_licence cc on cc.id=p.id +--join ${stats_db_name}.indi_pub_gold_oa ga on ga.id=p.id +--join ${stats_db_name}.result_instance ri on ri.id=p.id +--join ${stats_db_name}.datasource d on d.id=ri.hostedby +--where cc.has_cc_license=0 and ga.is_gold=0 +--and (d.type='Journal' or d.type='Journal Aggregator/Publisher') +--and ri.accessright='Open Access') tmp on tmp.id=p.id; + +create table ${stats_db_name}.indi_pub_bronze stored as parquet as +select pd.id,coalesce(is_bronze_oa,0) is_bronze_oa from ${stats_db_name}.publication_datasources pd +left outer join (select pd.id, 1 as is_bronze_oa from ${stats_db_name}.publication_datasources pd +join ${stats_db_name}.datasource d on pd.datasource=d.id +join ${stats_db_name}.result_instance ri on ri.id=pd.id +join ${stats_db_name}.indi_pub_gold_oa indi_gold on indi_gold.id=pd.id +join ${stats_db_name}.result_accessroute ra on ra.id=pd.id +where d.type like '%Journal%' and ri.accessright!='Closed Access' and (ri.accessright_uw!='gold' +or indi_gold.is_gold=0) and (ra.accessroute='bronze' or ri.license is null)) tmp +on pd.id=tmp.id; CREATE TEMPORARY TABLE ${stats_db_name}.project_year_result_year as select p.id project_id, acronym, r.id result_id, r.year, p.end_year @@ -709,6 +831,8 @@ join ${stats_db_name}.result_projects rp on p.id=rp.project join ${stats_db_name}.result r on r.id=rp.id where p.end_year is NOT NULL and r.year is not null; +drop table if exists ${stats_db_name}.indi_is_project_result_after purge; + create table if not exists ${stats_db_name}.indi_is_project_result_after stored as parquet as select pry.project_id, pry.acronym, pry.result_id, coalesce(is_project_result_after, 0) as is_project_result_after @@ -719,6 +843,8 @@ where pry.year>pry.end_year) tmp on pry.result_id=tmp.result_id; drop table ${stats_db_name}.project_year_result_year purge; +drop table ${stats_db_name}.indi_is_funder_plan_s purge; + create table if not exists ${stats_db_name}.indi_is_funder_plan_s stored as parquet as select distinct f.id, f.name, coalesce(is_funder_plan_s, 0) as is_funder_plan_s from ${stats_db_name}.funder f @@ -727,6 +853,7 @@ from ${stats_db_name}.funder f on f.name= tmp.name; --Funder Fairness +drop table ${stats_db_name}.indi_funder_fairness purge; create table if not exists ${stats_db_name}.indi_funder_fairness stored as parquet as with result_fair as @@ -745,6 +872,8 @@ from allresults join result_fair on result_fair.funder=allresults.funder; --RIs Fairness +drop table ${stats_db_name}.indi_ris_fairness purge; + create table if not exists ${stats_db_name}.indi_ris_fairness stored as parquet as with result_contexts as (select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc @@ -830,6 +959,8 @@ select software_oa.funder, software_oa.no_oasoftware/allsoftware.no_allsoftware from ${stats_db_name}.allsoftware join ${stats_db_name}.software_oa on allsoftware.funder=software_oa.funder; +drop table ${stats_db_name}.indi_funder_openess purge; + create table if not exists ${stats_db_name}.indi_funder_openess stored as parquet as select allpubsshare.funder, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) @@ -916,6 +1047,8 @@ select software_oa.ri_initiative, software_oa.no_oasoftware/allsoftware.no_allso from ${stats_db_name}.allsoftware join ${stats_db_name}.software_oa on allsoftware.ri_initiative=software_oa.ri_initiative; +drop table ${stats_db_name}.indi_ris_openess purge; + create table if not exists ${stats_db_name}.indi_ris_openess stored as parquet as select allpubsshare.ri_initiative, (p+if(isnull(s),0,s)+if(isnull(d),0,d))/(1+(case when s is null then 0 else 1 end) @@ -940,6 +1073,8 @@ DROP TABLE ${stats_db_name}.alldatasetssshare purge; DROP TABLE ${stats_db_name}.allsoftwaresshare purge; --Funder Findability +drop table ${stats_db_name}.indi_funder_findable purge; + create table if not exists ${stats_db_name}.indi_funder_findable stored as parquet as with result_findable as (select p.funder funder, count(distinct rp.id) no_result_findable from ${stats_db_name}.result_projects rp @@ -958,6 +1093,8 @@ from allresults join result_findable on result_findable.funder=allresults.funder; --RIs Findability +drop table ${stats_db_name}.indi_ris_findable purge; + create table if not exists ${stats_db_name}.indi_ris_findable stored as parquet as with result_contexts as (select distinct rc.id, context.name ri_initiative from ${stats_db_name}.result_concepts rc From 17586f0ff8d0e8d6225ecff52fa072ed4e66c3d4 Mon Sep 17 00:00:00 2001 From: dimitrispie Date: Mon, 9 Oct 2023 14:21:31 +0300 Subject: [PATCH 07/12] Update step20-createMonitorDB.sql Add result_orcid table to monitor dbs --- .../oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql index 586bee347..d5d242230 100644 --- a/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql +++ b/dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql @@ -162,6 +162,8 @@ create table TARGET.result_fos stored as parquet as select * from SOURCE.result_ create table TARGET.result_accessroute stored as parquet as select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result r where r.id=orig.id); --ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; +create table TARGET.result_orcid stored as parquet as select * from SOURCE.result_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id); + create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result); create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result); create table TARGET.result_result STORED AS PARQUET as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; From 9a98f408b36d6ebcd0b1bdeaaa64565c0a899f03 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 10 Oct 2023 09:36:11 +0200 Subject: [PATCH 08/12] code formatting --- .../opencitations/CreateActionSetSparkJob.java | 2 +- .../dnetlib/doiboost/crossref/Crossref2Oaf.scala | 16 +++++++++------- .../doiboost/crossref/CrossrefMappingTest.scala | 2 +- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index a367ba852..b707fdcd3 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.*; -import eu.dnetlib.dhp.schema.oaf.utils.*; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; @@ -30,6 +29,7 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.*; import eu.dnetlib.dhp.utils.DHPUtils; import scala.Tuple2; diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index e0fdb9ce4..565d34e62 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -31,9 +31,7 @@ case class mappingAuthor( affiliation: Option[mappingAffiliation] ) {} -case class funderInfo(id:String,uri:String, name:String,synonym:List[String] ) {} - - +case class funderInfo(id: String, uri: String, name: String, synonym: List[String]) {} case class mappingFunder(name: String, DOI: Option[String], award: Option[List[String]]) {} @@ -41,7 +39,9 @@ case object Crossref2Oaf { val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass) val irishFunder: List[funderInfo] = { - val s = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")).mkString + val s = Source + .fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")) + .mkString implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats lazy val json: org.json4s.JValue = parse(s) json.extract[List[funderInfo]] @@ -100,9 +100,11 @@ case object Crossref2Oaf { "report" -> "0017 Report" ) - def getIrishId(doi:String):Option[String] = { - val id =doi.split("/").last - irishFunder.find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))).map(f => f.id) + def getIrishId(doi: String): Option[String] = { + val id = doi.split("/").last + irishFunder + .find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))) + .map(f => f.id) } def mappingResult(result: Result, json: JValue, cobjCategory: String): Result = { diff --git a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala index 7961376c5..fbf6f72c0 100644 --- a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala +++ b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala @@ -50,7 +50,7 @@ class CrossrefMappingTest { } } - def checkRelation(generatedOAF: List[Oaf]): Unit = { + def checkRelation(generatedOAF: List[Oaf]): Unit = { val rels: List[Relation] = generatedOAF.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]] From 110ce4b40fc54c2d60fe8120927e76b84580b8c9 Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 10 Oct 2023 09:46:40 +0200 Subject: [PATCH 09/12] extend the fos model to include the level4 and the scores for level3 and level4. removed bip indicators from the instance --- .../dnetlib/dhp/actionmanager/Constants.java | 20 ++++-- .../GetFOSSparkJob.java | 9 ++- .../PrepareFOSSparkJob.java | 27 +++++-- .../SparkSaveUnresolved.java | 6 +- .../model/FOSDataModel.java | 63 +++++++++++++++-- .../CreateActionSetSparkJob.java | 2 +- .../oozie_app/workflow.xml | 56 +++++++-------- .../createunresolvedentities/GetFosTest.java | 39 +++++++++-- .../createunresolvedentities/PrepareTest.java | 70 +++++++++++++++++++ .../createunresolvedentities/ProduceTest.java | 34 +++++++++ .../createunresolvedentities/fos/fos_sbs2.csv | 26 +++++++ .../fos/fos_sbs_2.json | 25 +++++++ .../doiboost/crossref/Crossref2Oaf.scala | 16 +++-- .../crossref/CrossrefMappingTest.scala | 2 +- 14 files changed, 334 insertions(+), 61 deletions(-) create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv create mode 100644 dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java index 62556b16b..006d3af76 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/Constants.java @@ -40,6 +40,7 @@ public class Constants { public static final String SDG_CLASS_NAME = "Sustainable Development Goals"; public static final String NULL = "NULL"; + public static final String NA = "N/A"; public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -61,10 +62,16 @@ public class Constants { .map((MapFunction) value -> OBJECT_MAPPER.readValue(value, clazz), Encoders.bean(clazz)); } - public static Subject getSubject(String sbj, String classid, String classname, - String diqualifierclassid) { - if (sbj == null || sbj.equals(NULL)) + public static Subject getSubject(String sbj, String classid, String classname, String diqualifierclassid, + Boolean split) { + if (sbj == null || sbj.equals(NULL) || sbj.startsWith(NA)) return null; + String trust = ""; + String subject = sbj; + if (split) { + sbj = subject.split("@@")[0]; + trust = subject.split("@@")[1]; + } Subject s = new Subject(); s.setValue(sbj); s @@ -89,9 +96,14 @@ public class Constants { UPDATE_CLASS_NAME, ModelConstants.DNET_PROVENANCE_ACTIONS, ModelConstants.DNET_PROVENANCE_ACTIONS), - "")); + trust)); return s; + } + + public static Subject getSubject(String sbj, String classid, String classname, + String diqualifierclassid) { + return getSubject(sbj, classid, classname, diqualifierclassid, false); } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java index 0cc2f93df..abea6acd7 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFOSSparkJob.java @@ -75,9 +75,12 @@ public class GetFOSSparkJob implements Serializable { fosData.map((MapFunction) r -> { FOSDataModel fosDataModel = new FOSDataModel(); fosDataModel.setDoi(r.getString(0).toLowerCase()); - fosDataModel.setLevel1(r.getString(1)); - fosDataModel.setLevel2(r.getString(2)); - fosDataModel.setLevel3(r.getString(3)); + fosDataModel.setLevel1(r.getString(2)); + fosDataModel.setLevel2(r.getString(3)); + fosDataModel.setLevel3(r.getString(4)); + fosDataModel.setLevel4(r.getString(5)); + fosDataModel.setScoreL3(String.valueOf(r.getDouble(6))); + fosDataModel.setScoreL4(String.valueOf(r.getDouble(7))); return fosDataModel; }, Encoders.bean(FOSDataModel.class)) .write() diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java index 4d2d25215..57ad8b96a 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareFOSSparkJob.java @@ -78,12 +78,20 @@ public class PrepareFOSSparkJob implements Serializable { HashSet level1 = new HashSet<>(); HashSet level2 = new HashSet<>(); HashSet level3 = new HashSet<>(); - addLevels(level1, level2, level3, first); - it.forEachRemaining(v -> addLevels(level1, level2, level3, v)); + HashSet level4 = new HashSet<>(); + addLevels(level1, level2, level3, level4, first); + it.forEachRemaining(v -> addLevels(level1, level2, level3, level4, v)); List sbjs = new ArrayList<>(); - level1.forEach(l -> sbjs.add(getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); - level2.forEach(l -> sbjs.add(getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); - level3.forEach(l -> sbjs.add(getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); + level1 + .forEach(l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); + level2 + .forEach(l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID))); + level3 + .forEach( + l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID, true))); + level4 + .forEach( + l -> add(sbjs, getSubject(l, FOS_CLASS_ID, FOS_CLASS_NAME, UPDATE_SUBJECT_FOS_CLASS_ID, true))); r.setSubject(sbjs); r .setDataInfo( @@ -106,11 +114,18 @@ public class PrepareFOSSparkJob implements Serializable { .json(outputPath + "/fos"); } + private static void add(List sbsjs, Subject sbj) { + if (sbj != null) + sbsjs.add(sbj); + } + private static void addLevels(HashSet level1, HashSet level2, HashSet level3, + HashSet level4, FOSDataModel first) { level1.add(first.getLevel1()); level2.add(first.getLevel2()); - level3.add(first.getLevel3()); + level3.add(first.getLevel3() + "@@" + first.getScoreL3()); + level4.add(first.getLevel4() + "@@" + first.getScoreL4()); } } diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java index 3b9775094..93bbfcc88 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/SparkSaveUnresolved.java @@ -69,9 +69,9 @@ public class SparkSaveUnresolved implements Serializable { .mapGroups((MapGroupsFunction) (k, it) -> { Result ret = it.next(); it.forEachRemaining(r -> { - if (r.getInstance() != null) { - ret.setInstance(r.getInstance()); - } +// if (r.getInstance() != null) { +// ret.setInstance(r.getInstance()); +// } if (r.getSubject() != null) { if (ret.getSubject() != null) ret.getSubject().addAll(r.getSubject()); diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java index e98ba74a1..a82d7bfd6 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/model/FOSDataModel.java @@ -11,21 +11,43 @@ public class FOSDataModel implements Serializable { private String doi; @CsvBindByPosition(position = 1) +// @CsvBindByName(column = "doi") + private String oaid; + @CsvBindByPosition(position = 2) // @CsvBindByName(column = "level1") private String level1; - @CsvBindByPosition(position = 2) + @CsvBindByPosition(position = 3) // @CsvBindByName(column = "level2") private String level2; - @CsvBindByPosition(position = 3) + @CsvBindByPosition(position = 4) // @CsvBindByName(column = "level3") private String level3; + @CsvBindByPosition(position = 5) +// @CsvBindByName(column = "level3") + private String level4; + @CsvBindByPosition(position = 6) + private String scoreL3; + @CsvBindByPosition(position = 7) + private String scoreL4; + public FOSDataModel() { } + public FOSDataModel(String doi, String level1, String level2, String level3, String level4, String l3score, + String l4score) { + this.doi = doi; + this.level1 = level1; + this.level2 = level2; + this.level3 = level3; + this.level4 = level4; + this.scoreL3 = l3score; + this.scoreL4 = l4score; + } + public FOSDataModel(String doi, String level1, String level2, String level3) { this.doi = doi; this.level1 = level1; @@ -33,8 +55,41 @@ public class FOSDataModel implements Serializable { this.level3 = level3; } - public static FOSDataModel newInstance(String d, String level1, String level2, String level3) { - return new FOSDataModel(d, level1, level2, level3); + public static FOSDataModel newInstance(String d, String level1, String level2, String level3, String level4, + String scorel3, String scorel4) { + return new FOSDataModel(d, level1, level2, level3, level4, scorel3, scorel4); + } + + public String getOaid() { + return oaid; + } + + public void setOaid(String oaid) { + this.oaid = oaid; + } + + public String getLevel4() { + return level4; + } + + public void setLevel4(String level4) { + this.level4 = level4; + } + + public String getScoreL3() { + return scoreL3; + } + + public void setScoreL3(String scoreL3) { + this.scoreL3 = scoreL3; + } + + public String getScoreL4() { + return scoreL4; + } + + public void setScoreL4(String scoreL4) { + this.scoreL4 = scoreL4; } public String getDoi() { diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java index a367ba852..b707fdcd3 100644 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java +++ b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/opencitations/CreateActionSetSparkJob.java @@ -7,7 +7,6 @@ import java.io.IOException; import java.io.Serializable; import java.util.*; -import eu.dnetlib.dhp.schema.oaf.utils.*; import org.apache.commons.cli.ParseException; import org.apache.commons.io.IOUtils; import org.apache.hadoop.io.Text; @@ -30,6 +29,7 @@ import eu.dnetlib.dhp.application.ArgumentApplicationParser; import eu.dnetlib.dhp.schema.action.AtomicAction; import eu.dnetlib.dhp.schema.common.ModelConstants; import eu.dnetlib.dhp.schema.oaf.*; +import eu.dnetlib.dhp.schema.oaf.utils.*; import eu.dnetlib.dhp.utils.DHPUtils; import scala.Tuple2; diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml index c8af64594..a2935a71d 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml @@ -6,10 +6,10 @@ the input path of the resources to be extended - - bipScorePath - the path where to find the bipFinder scores - + + + + outputPath the path where to store the actionset @@ -77,34 +77,34 @@ - + - - - yarn - cluster - Produces the unresolved from BIP! Finder - eu.dnetlib.dhp.actionmanager.createunresolvedentities.PrepareBipFinder - dhp-aggregation-${projectVersion}.jar - - --executor-memory=${sparkExecutorMemory} - --executor-cores=${sparkExecutorCores} - --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} - --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} - --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} - --conf spark.sql.warehouse.dir=${sparkSqlWarehouseDir} - - --sourcePath${bipScorePath} - --outputPath${workingDir}/prepared - - - - + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java index 7e0acc2bb..d4fe129df 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/GetFosTest.java @@ -13,10 +13,7 @@ import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaRDD; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.sql.SparkSession; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +65,7 @@ public class GetFosTest { } @Test + @Disabled void test3() throws Exception { final String sourcePath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs.tsv") @@ -96,4 +94,37 @@ public class GetFosTest { tmp.foreach(t -> Assertions.assertTrue(t.getLevel3() != null)); } + + @Test + void test4() throws Exception { + final String sourcePath = getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv") + .getPath(); + + final String outputPath = workingDir.toString() + "/fos.json"; + GetFOSSparkJob + .main( + new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--sourcePath", sourcePath, + "--delimiter", ",", + "-outputPath", outputPath + + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(outputPath) + .map(item -> OBJECT_MAPPER.readValue(item, FOSDataModel.class)); + + tmp.foreach(t -> Assertions.assertTrue(t.getDoi() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getLevel1() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getLevel2() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getLevel3() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getLevel4() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getScoreL3() != null)); + tmp.foreach(t -> Assertions.assertTrue(t.getScoreL4() != null)); + + } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java index cc8108bde..ccb0ebbff 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java @@ -222,6 +222,76 @@ public class PrepareTest { } + @Test + void fosPrepareTest2() throws Exception { + final String sourcePath = getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json") + .getPath(); + + PrepareFOSSparkJob + .main( + new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--sourcePath", sourcePath, + + "-outputPath", workingDir.toString() + "/work" + + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/work/fos") + .map(item -> OBJECT_MAPPER.readValue(item, Result.class)); + + String doi1 = "unresolved::10.1016/j.revmed.2006.07.012::doi"; + + assertEquals(13, tmp.count()); + assertEquals(1, tmp.filter(row -> row.getId().equals(doi1)).count()); + + Result result = tmp + .filter(r -> r.getId().equals(doi1)) + .first(); + + result.getSubject().forEach(s -> System.out.println(s.getValue() + " trust = " + s.getDataInfo().getTrust())); + Assertions.assertEquals(6, result.getSubject().size()); + + assertTrue( + result + .getSubject() + .stream() + .anyMatch( + s -> s.getValue().contains("03 medical and health sciences") + && s.getDataInfo().getTrust().equals(""))); + + assertTrue( + result + .getSubject() + .stream() + .anyMatch( + s -> s.getValue().contains("0302 clinical medicine") && s.getDataInfo().getTrust().equals(""))); + + assertTrue( + result + .getSubject() + .stream() + .anyMatch( + s -> s + .getValue() + .contains("030204 cardiovascular system & hematology") + && s.getDataInfo().getTrust().equals("0.5101401805877686"))); + assertTrue( + result + .getSubject() + .stream() + .anyMatch( + s -> s + .getValue() + .contains("03020409 Hematology/Coagulopathies") + && s.getDataInfo().getTrust().equals("0.0546871414174914"))); + + } + @Test void sdgPrepareTest() throws Exception { final String sourcePath = getClass() diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java index c3c110f09..fce6c1e97 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java @@ -379,6 +379,40 @@ public class ProduceTest { .map(item -> OBJECT_MAPPER.readValue(item, Result.class)); } + @Test + public JavaRDD getResultFosJavaRDD() throws Exception { + + final String fosPath = getClass() + .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json") + .getPath(); + + PrepareFOSSparkJob + .main( + new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--sourcePath", fosPath, + "-outputPath", workingDir.toString() + "/work" + }); + + SparkSaveUnresolved.main(new String[] { + "--isSparkSessionManaged", Boolean.FALSE.toString(), + "--sourcePath", workingDir.toString() + "/work", + + "-outputPath", workingDir.toString() + "/unresolved" + + }); + + final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); + + JavaRDD tmp = sc + .textFile(workingDir.toString() + "/unresolved") + .map(item -> OBJECT_MAPPER.readValue(item, Result.class)); + tmp.foreach(r -> System.out.println(new ObjectMapper().writeValueAsString(r))); + + return tmp; + + } + @Test void prepareTest5Subjects() throws Exception { final String doi = "unresolved::10.1063/5.0032658::doi"; diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv new file mode 100644 index 000000000..3b1f2304f --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs2.csv @@ -0,0 +1,26 @@ +DOI,OAID,level1,level2,level3,level4,score_for_L3,score_for_L4 +10.1016/j.anucene.2006.02.004,doi_________::00059d9963edf633bec756fb21b5bd72,02 engineering and technology,"0202 electrical engineering, electronic engineering, information engineering",020209 energy,02020908 Climate change policy/Ethanol fuel,0.5,0.5 +10.1016/j.anucene.2006.02.004,doi_________::00059d9963edf633bec756fb21b5bd72,02 engineering and technology,0211 other engineering and technologies,021108 energy,02110808 Climate change policy/Ethanol fuel,0.5,0.5 +10.1016/j.revmed.2006.07.010,doi_________::0026476c1651a92c933d752ff12496c7,03 medical and health sciences,0302 clinical medicine,030220 oncology & carcinogenesis,N/A,0.5036656856536865,0.0 +10.1016/j.revmed.2006.07.010,doi_________::0026476c1651a92c933d752ff12496c7,03 medical and health sciences,0302 clinical medicine,030212 general & internal medicine,N/A,0.4963343143463135,0.0 +10.20965/jrm.2006.p0312,doi_________::0028336a2f3826cc83c47dbefac71543,02 engineering and technology,0209 industrial biotechnology,020901 industrial engineering & automation,02090104 Robotics/Robots,0.6111094951629639,0.5053805979936855 +10.20965/jrm.2006.p0312,doi_________::0028336a2f3826cc83c47dbefac71543,01 natural sciences,0104 chemical sciences,010401 analytical chemistry,N/A,0.3888905048370361,0.0 +10.1111/j.1747-7379.2006.040_1.x,doi_________::002c7077e7c114a8304eb90f59e45fa4,05 social sciences,0506 political science,050602 political science & public administration,05060202 Ethnic groups/Ethnicity,0.6159052848815918,0.7369035568037298 +10.1111/j.1747-7379.2006.040_1.x,doi_________::002c7077e7c114a8304eb90f59e45fa4,05 social sciences,0502 economics and business,050207 economics,N/A,0.3840946555137634,0.0 +10.1007/s10512-006-0049-9,doi_________::003f29f9254819cf4c78558b1bc25f10,02 engineering and technology,"0202 electrical engineering, electronic engineering, information engineering",020209 energy,02020908 Climate change policy/Ethanol fuel,0.5,0.5 +10.1007/s10512-006-0049-9,doi_________::003f29f9254819cf4c78558b1bc25f10,02 engineering and technology,0211 other engineering and technologies,021108 energy,02110808 Climate change policy/Ethanol fuel,0.5,0.5 +10.1111/j.1365-2621.2005.01045.x,doi_________::00419355b4c3e0646bd0e1b301164c8e,04 agricultural and veterinary sciences,0404 agricultural biotechnology,040401 food science,04040102 Food science/Food industry,0.5,0.5 +10.1111/j.1365-2621.2005.01045.x,doi_________::00419355b4c3e0646bd0e1b301164c8e,04 agricultural and veterinary sciences,0405 other agricultural sciences,040502 food science,04050202 Food science/Food industry,0.5,0.5 +10.1002/chin.200617262,doi_________::004c8cef80668904961b9e62841793c8,01 natural sciences,0104 chemical sciences,010405 organic chemistry,01040508 Functional groups/Ethers,0.5566747188568115,0.5582916736602783 +10.1002/chin.200617262,doi_________::004c8cef80668904961b9e62841793c8,01 natural sciences,0104 chemical sciences,010402 general chemistry,01040207 Chemical synthesis/Total synthesis,0.4433253407478332,0.4417082965373993 +10.1016/j.revmed.2006.07.012,doi_________::005b1d0fb650b680abaf6cfe26a21604,03 medical and health sciences,0302 clinical medicine,030204 cardiovascular system & hematology,03020409 Hematology/Coagulopathies,0.5101401805877686,0.0546871414174914 +10.1016/j.revmed.2006.07.012,doi_________::005b1d0fb650b680abaf6cfe26a21604,03 medical and health sciences,0301 basic medicine,030105 genetics & heredity,N/A,0.4898599088191986,0.0 +10.4109/jslab.17.132,doi_________::00889baa06de363e37930daaf8e800c0,03 medical and health sciences,0301 basic medicine,030104 developmental biology,N/A,0.5,0.0 +10.4109/jslab.17.132,doi_________::00889baa06de363e37930daaf8e800c0,03 medical and health sciences,0303 health sciences,030304 developmental biology,N/A,0.5,0.0 +10.1108/00251740610715687,doi_________::0092cb1b1920d556719385a26363ecaa,05 social sciences,0502 economics and business,050203 business & management,05020311 International business/International trade,0.605047881603241,0.2156608108845153 +10.1108/00251740610715687,doi_________::0092cb1b1920d556719385a26363ecaa,05 social sciences,0502 economics and business,050211 marketing,N/A,0.394952118396759,0.0 +10.1080/03067310500248098,doi_________::00a76678d230e3f20b6356804448028f,04 agricultural and veterinary sciences,0404 agricultural biotechnology,040401 food science,04040102 Food science/Food industry,0.5,0.5 +10.1080/03067310500248098,doi_________::00a76678d230e3f20b6356804448028f,04 agricultural and veterinary sciences,0405 other agricultural sciences,040502 food science,04050202 Food science/Food industry,0.5,0.5 +10.3152/147154306781778533,doi_________::00acc520f3939e5a6675343881fed4f2,05 social sciences,0502 economics and business,050203 business & management,05020307 Innovation/Product management,0.5293408632278442,0.5326762795448303 +10.3152/147154306781778533,doi_________::00acc520f3939e5a6675343881fed4f2,05 social sciences,0509 other social sciences,050905 science studies,05090502 Social philosophy/Capitalism,0.4706590473651886,0.4673237204551697 +10.1785/0120050806,doi_________::00d5831d329e7ae4523d78bfc3042e98,02 engineering and technology,0211 other engineering and technologies,021101 geological & geomatics engineering,02110103 Concrete/Building materials,0.5343400835990906,0.3285667930180677 \ No newline at end of file diff --git a/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json new file mode 100644 index 000000000..00ffad70c --- /dev/null +++ b/dhp-workflows/dhp-aggregation/src/test/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos_sbs_2.json @@ -0,0 +1,25 @@ +{"doi":"10.1016/j.anucene.2006.02.004","level1":"02 engineering and technology","level2":"0202 electrical engineering, electronic engineering, information engineering","level3":"020209 energy","level4":"02020908 Climate change policy/Ethanol fuel","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1016/j.anucene.2006.02.004","level1":"02 engineering and technology","level2":"0211 other engineering and technologies","level3":"021108 energy","level4":"02110808 Climate change policy/Ethanol fuel","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1016/j.revmed.2006.07.010","level1":"03 medical and health sciences","level2":"0302 clinical medicine","level3":"030220 oncology & carcinogenesis","level4":"N/A","scoreL3":"0.5036656856536865","scoreL4":"0.0"} +{"doi":"10.1016/j.revmed.2006.07.010","level1":"03 medical and health sciences","level2":"0302 clinical medicine","level3":"030212 general & internal medicine","level4":"N/A","scoreL3":"0.4963343143463135","scoreL4":"0.0"} +{"doi":"10.20965/jrm.2006.p0312","level1":"02 engineering and technology","level2":"0209 industrial biotechnology","level3":"020901 industrial engineering & automation","level4":"02090104 Robotics/Robots","scoreL3":"0.6111094951629639","scoreL4":"0.5053805979936855"} +{"doi":"10.20965/jrm.2006.p0312","level1":"01 natural sciences","level2":"0104 chemical sciences","level3":"010401 analytical chemistry","level4":"N/A","scoreL3":"0.3888905048370361","scoreL4":"0.0"} +{"doi":"10.1111/j.1747-7379.2006.040_1.x","level1":"05 social sciences","level2":"0506 political science","level3":"050602 political science & public administration","level4":"05060202 Ethnic groups/Ethnicity","scoreL3":"0.6159052848815918","scoreL4":"0.7369035568037298"} +{"doi":"10.1111/j.1747-7379.2006.040_1.x","level1":"05 social sciences","level2":"0502 economics and business","level3":"050207 economics","level4":"N/A","scoreL3":"0.3840946555137634","scoreL4":"0.0"} +{"doi":"10.1007/s10512-006-0049-9","level1":"02 engineering and technology","level2":"0202 electrical engineering, electronic engineering, information engineering","level3":"020209 energy","level4":"02020908 Climate change policy/Ethanol fuel","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1007/s10512-006-0049-9","level1":"02 engineering and technology","level2":"0211 other engineering and technologies","level3":"021108 energy","level4":"02110808 Climate change policy/Ethanol fuel","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1111/j.1365-2621.2005.01045.x","level1":"04 agricultural and veterinary sciences","level2":"0404 agricultural biotechnology","level3":"040401 food science","level4":"04040102 Food science/Food industry","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1111/j.1365-2621.2005.01045.x","level1":"04 agricultural and veterinary sciences","level2":"0405 other agricultural sciences","level3":"040502 food science","level4":"04050202 Food science/Food industry","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1002/chin.200617262","level1":"01 natural sciences","level2":"0104 chemical sciences","level3":"010405 organic chemistry","level4":"01040508 Functional groups/Ethers","scoreL3":"0.5566747188568115","scoreL4":"0.5582916736602783"} +{"doi":"10.1002/chin.200617262","level1":"01 natural sciences","level2":"0104 chemical sciences","level3":"010402 general chemistry","level4":"01040207 Chemical synthesis/Total synthesis","scoreL3":"0.4433253407478332","scoreL4":"0.4417082965373993"} +{"doi":"10.1016/j.revmed.2006.07.012","level1":"03 medical and health sciences","level2":"0302 clinical medicine","level3":"030204 cardiovascular system & hematology","level4":"03020409 Hematology/Coagulopathies","scoreL3":"0.5101401805877686","scoreL4":"0.0546871414174914"} +{"doi":"10.1016/j.revmed.2006.07.012","level1":"03 medical and health sciences","level2":"0301 basic medicine","level3":"030105 genetics & heredity","level4":"N/A","scoreL3":"0.4898599088191986","scoreL4":"0.0"} +{"doi":"10.4109/jslab.17.132","level1":"03 medical and health sciences","level2":"0301 basic medicine","level3":"030104 developmental biology","level4":"N/A","scoreL3":"0.5","scoreL4":"0.0"} +{"doi":"10.4109/jslab.17.132","level1":"03 medical and health sciences","level2":"0303 health sciences","level3":"030304 developmental biology","level4":"N/A","scoreL3":"0.5","scoreL4":"0.0"} +{"doi":"10.1108/00251740610715687","level1":"05 social sciences","level2":"0502 economics and business","level3":"050203 business & management","level4":"05020311 International business/International trade","scoreL3":"0.605047881603241","scoreL4":"0.2156608108845153"} +{"doi":"10.1108/00251740610715687","level1":"05 social sciences","level2":"0502 economics and business","level3":"050211 marketing","level4":"N/A","scoreL3":"0.394952118396759","scoreL4":"0.0"} +{"doi":"10.1080/03067310500248098","level1":"04 agricultural and veterinary sciences","level2":"0404 agricultural biotechnology","level3":"040401 food science","level4":"04040102 Food science/Food industry","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.1080/03067310500248098","level1":"04 agricultural and veterinary sciences","level2":"0405 other agricultural sciences","level3":"040502 food science","level4":"04050202 Food science/Food industry","scoreL3":"0.5","scoreL4":"0.5"} +{"doi":"10.3152/147154306781778533","level1":"05 social sciences","level2":"0502 economics and business","level3":"050203 business & management","level4":"05020307 Innovation/Product management","scoreL3":"0.5293408632278442","scoreL4":"0.5326762795448303"} +{"doi":"10.3152/147154306781778533","level1":"05 social sciences","level2":"0509 other social sciences","level3":"050905 science studies","level4":"05090502 Social philosophy/Capitalism","scoreL3":"0.4706590473651886","scoreL4":"0.4673237204551697"} +{"doi":"10.1785/0120050806","level1":"02 engineering and technology","level2":"0211 other engineering and technologies","level3":"021101 geological & geomatics engineering","level4":"02110103 Concrete/Building materials","scoreL3":"0.5343400835990906","scoreL4":"0.3285667930180677"} diff --git a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala index e0fdb9ce4..565d34e62 100644 --- a/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala +++ b/dhp-workflows/dhp-doiboost/src/main/scala/eu/dnetlib/doiboost/crossref/Crossref2Oaf.scala @@ -31,9 +31,7 @@ case class mappingAuthor( affiliation: Option[mappingAffiliation] ) {} -case class funderInfo(id:String,uri:String, name:String,synonym:List[String] ) {} - - +case class funderInfo(id: String, uri: String, name: String, synonym: List[String]) {} case class mappingFunder(name: String, DOI: Option[String], award: Option[List[String]]) {} @@ -41,7 +39,9 @@ case object Crossref2Oaf { val logger: Logger = LoggerFactory.getLogger(Crossref2Oaf.getClass) val irishFunder: List[funderInfo] = { - val s = Source.fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")).mkString + val s = Source + .fromInputStream(getClass.getResourceAsStream("/eu/dnetlib/dhp/doiboost/crossref/irish_funder.json")) + .mkString implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats lazy val json: org.json4s.JValue = parse(s) json.extract[List[funderInfo]] @@ -100,9 +100,11 @@ case object Crossref2Oaf { "report" -> "0017 Report" ) - def getIrishId(doi:String):Option[String] = { - val id =doi.split("/").last - irishFunder.find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))).map(f => f.id) + def getIrishId(doi: String): Option[String] = { + val id = doi.split("/").last + irishFunder + .find(f => id.equalsIgnoreCase(f.id) || (f.synonym.nonEmpty && f.synonym.exists(s => s.equalsIgnoreCase(id)))) + .map(f => f.id) } def mappingResult(result: Result, json: JValue, cobjCategory: String): Result = { diff --git a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala index 7961376c5..fbf6f72c0 100644 --- a/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala +++ b/dhp-workflows/dhp-doiboost/src/test/scala/eu/dnetlib/dhp/doiboost/crossref/CrossrefMappingTest.scala @@ -50,7 +50,7 @@ class CrossrefMappingTest { } } - def checkRelation(generatedOAF: List[Oaf]): Unit = { + def checkRelation(generatedOAF: List[Oaf]): Unit = { val rels: List[Relation] = generatedOAF.filter(p => p.isInstanceOf[Relation]).asInstanceOf[List[Relation]] From ed9282ef2a3e40a76308fbff9226a3bfa6a90df6 Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 10 Oct 2023 09:52:03 +0200 Subject: [PATCH 10/12] removed module dhp-stats-monitor-update --- .../oozie_app/config-default.xml | 30 ---- .../oozie_app/copyDataToImpalaCluster.sh | 75 --------- .../oozie_app/finalizeImpalaCluster.sh | 29 ---- .../graph/stats-monitor/oozie_app/monitor.sh | 54 ------- .../oozie_app/scripts/updateMonitorDB.sql | 138 ---------------- .../oozie_app/scripts/updateMonitorDBAll.sql | 150 ------------------ .../scripts/updateMonitorDB_institutions.sql | 12 -- .../stats-monitor/oozie_app/workflow.xml | 110 ------------- 8 files changed, 598 deletions(-) delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql delete mode 100644 dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml deleted file mode 100644 index b2a1322e6..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/config-default.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - jobTracker - ${jobTracker} - - - nameNode - ${nameNode} - - - oozie.use.system.libpath - true - - - oozie.action.sharelib.for.spark - spark2 - - - hive_metastore_uris - thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083 - - - hive_jdbc_url - jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1;?spark.executor.memory=22166291558;spark.yarn.executor.memoryOverhead=3225;spark.driver.memory=15596411699;spark.yarn.driver.memoryOverhead=1228 - - - oozie.wf.workflow.notification.url - {serviceUrl}/v1/oozieNotification/jobUpdate?jobId=$jobId%26status=$status - - \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh deleted file mode 100644 index 1587f7152..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/copyDataToImpalaCluster.sh +++ /dev/null @@ -1,75 +0,0 @@ -export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs -export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) -if ! [ -L $link_folder ] -then - rm -Rf "$link_folder" - ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} -fi - -#export HADOOP_USER_NAME=$2 - -function copydb() { - - export HADOOP_USER="dimitris.pierrakos" - export HADOOP_USER_NAME='dimitris.pierrakos' - - db=$1 - FILE=("hive_wf_tmp_"$RANDOM) - hdfs dfs -mkdir hdfs://impala-cluster-mn1.openaire.eu:8020/tmp/$FILE/ - - # change ownership to impala -# hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/${db}.db - hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/ - - - # copy the databases from ocean to impala - echo "copying $db" - hadoop distcp -Dmapreduce.map.memory.mb=6144 -pb hdfs://nameservice1/user/hive/warehouse/${db}.db hdfs://impala-cluster-mn1.openaire.eu:8020/tmp/$FILE/ - - hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -chmod -R 777 /tmp/$FILE/${db}.db - - # drop tables from db - for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; - do - `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop table $i;"`; - done - - # drop views from db - for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; - do - `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "drop view $i;"`; - done - - # delete the database - impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database if exists ${db} cascade"; - - # create the databases - impala-shell -i impala-cluster-dn1.openaire.eu -q "create database ${db}"; - - impala-shell -q "INVALIDATE METADATA" - echo "creating schema for ${db}" - for (( k = 0; k < 5; k ++ )); do - for i in `impala-shell -d ${db} --delimited -q "show tables"`; - do - impala-shell -d ${db} --delimited -q "show create table $i"; - done | sed 's/"$/;/' | sed 's/^"//' | sed 's/[[:space:]]\date[[:space:]]/`date`/g' | impala-shell --user $HADOOP_USER_NAME -i impala-cluster-dn1.openaire.eu -c -f - - done - - # load the data from /tmp in the respective tables - echo "copying data in tables and computing stats" - for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} --delimited -q "show tables"`; - do - impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "load data inpath '/tmp/$FILE/${db}.db/$i' into table $i"; - impala-shell -i impala-cluster-dn1.openaire.eu -d ${db} -q "compute stats $i"; - done - - # deleting the remaining directory from hdfs -hdfs dfs -conf /etc/impala_cluster/hdfs-site.xml -rm -R /tmp/$FILE/${db}.db -} - -MONITOR_DB=$1 -#HADOOP_USER_NAME=$2 - -copydb $MONITOR_DB'_institutions' -copydb $MONITOR_DB - diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh deleted file mode 100644 index a7227e0c8..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/finalizeImpalaCluster.sh +++ /dev/null @@ -1,29 +0,0 @@ -export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs -export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) -if ! [ -L $link_folder ] -then - rm -Rf "$link_folder" - ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} -fi - -function createShadowDB() { - SOURCE=$1 - SHADOW=$2 - - # drop views from db - for i in `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} --delimited -q "show tables"`; - do - `impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "drop view $i;"`; - done - - impala-shell -i impala-cluster-dn1.openaire.eu -q "drop database ${SHADOW} CASCADE"; - impala-shell -i impala-cluster-dn1.openaire.eu -q "create database if not exists ${SHADOW}"; -# impala-shell -i impala-cluster-dn1.openaire.eu -d ${SHADOW} -q "show tables" | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - - impala-shell -i impala-cluster-dn1.openaire.eu -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -i impala-cluster-dn1.openaire.eu -f - -} - -MONITOR_DB=$1 -MONITOR_DB_SHADOW=$2 - -createShadowDB $MONITOR_DB'_institutions' $MONITOR_DB'_institutions_shadow' -createShadowDB $MONITOR_DB $MONITOR_DB'_shadow' diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh deleted file mode 100644 index 4f1889c9e..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/monitor.sh +++ /dev/null @@ -1,54 +0,0 @@ -export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs -export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami) -if ! [ -L $link_folder ] -then - rm -Rf "$link_folder" - ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder} -fi - -export SOURCE=$1 -export TARGET=$2 -export SHADOW=$3 -export SCRIPT_PATH=$4 -export SCRIPT_PATH2=$5 -export SCRIPT_PATH2=$6 - -export HIVE_OPTS="-hiveconf mapred.job.queue.name=analytics -hiveconf hive.spark.client.connect.timeout=120000ms -hiveconf hive.spark.client.server.connect.timeout=300000ms -hiveconf spark.executor.memory=19166291558 -hiveconf spark.yarn.executor.memoryOverhead=3225 -hiveconf spark.driver.memory=11596411699 -hiveconf spark.yarn.driver.memoryOverhead=1228" -export HADOOP_USER_NAME="oozie" - -echo "Getting file from " $4 -hdfs dfs -copyToLocal $4 - -echo "Getting file from " $5 -hdfs dfs -copyToLocal $5 - -echo "Getting file from " $6 -hdfs dfs -copyToLocal $6 - -#update Institutions DB -cat updateMonitorDB_institutions.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo -hive $HIVE_OPTS -f foo -cat updateMonitorDB.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2_institutions/g1" > foo -hive $HIVE_OPTS -f foo - -echo "Hive shell finished" - -echo "Updating shadow monitor insitutions database" -hive -e "drop database if exists ${SHADOW}_institutions cascade" -hive -e "create database if not exists ${SHADOW}_institutions" -hive $HIVE_OPTS --database ${2}_institutions -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}_institutions.\1 as select * from ${2}_institutions.\1;/" > foo -hive -f foo -echo "Shadow db monitor insitutions ready!" - -#update Monitor DB -cat updateMonitorDBAll.sql | sed "s/SOURCE/$1/g" | sed "s/TARGET/$2/g1" > foo -hive $HIVE_OPTS -f foo - -echo "Hive shell finished" - -echo "Updating shadow monitor database" -hive -e "drop database if exists ${SHADOW} cascade" -hive -e "create database if not exists ${SHADOW}" -hive $HIVE_OPTS --database ${2} -e "show tables" | grep -v WARN | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${2}.\1;/" > foo -hive -f foo -echo "Shadow db monitor insitutions ready!" diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql deleted file mode 100644 index 248b7e564..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB.sql +++ /dev/null @@ -1,138 +0,0 @@ -INSERT INTO TARGET.result select * from TARGET.result_new; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; - -INSERT INTO TARGET.result_citations select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; - -INSERT INTO TARGET.result_references_oc select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; - -INSERT INTO TARGET.result_classifications select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; - -INSERT INTO TARGET.result_apc select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; - -INSERT INTO TARGET.result_concepts select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; - -INSERT INTO TARGET.result_datasources select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; - -INSERT INTO TARGET.result_fundercount select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; - -INSERT INTO TARGET.result_gold select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; - -INSERT INTO TARGET.result_greenoa select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; - -INSERT INTO TARGET.result_languages select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; - -INSERT INTO TARGET.result_licenses select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; - -INSERT INTO TARGET.result_oids select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; - -INSERT INTO TARGET.result_organization select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; - -INSERT INTO TARGET.result_peerreviewed select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; - -INSERT INTO TARGET.result_pids select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; - -INSERT INTO TARGET.result_projectcount select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; - -INSERT INTO TARGET.result_projects select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; - -INSERT INTO TARGET.result_refereed select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; - -INSERT INTO TARGET.result_sources select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; - -INSERT INTO TARGET.result_topics select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; - -INSERT INTO TARGET.result_fos select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; - -INSERT INTO TARGET.result_accessroute select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; - -create or replace view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result_new); -create or replace view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result_new); -insert into TARGET.result_result select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; -drop view TARGET.foo1; -drop view TARGET.foo2; -ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; - - --- indicators --- Sprint 1 ---- -INSERT INTO TARGET.indi_pub_green_oa select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_grey_lit select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_doi_from_crossref select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; --- Sprint 2 ---- -INSERT INTO TARGET.indi_result_has_cc_licence select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_has_cc_licence_url select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_has_abstract select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_with_orcid select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; ----- Sprint 3 ---- -INSERT INTO TARGET.indi_funded_result_with_fundref select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; - ----- Sprint 4 ---- -INSERT INTO TARGET.indi_pub_diamond select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_in_transformative select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_closed_other_open select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; ----- Sprint 5 ---- -INSERT INTO TARGET.indi_result_no_of_copies select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; ----- Sprint 6 ---- -INSERT INTO TARGET.indi_pub_hybrid_oa_with_cc select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_bronze_oa select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_datasource select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_year select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_datasource_year select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; ----- Sprint 7 ---- -INSERT INTO TARGET.indi_pub_gold_oa select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_hybrid select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_has_preprint select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_in_subscribed select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_with_pid select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; -INSERT INTO TARGET.indi_impact_measures select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_interdisciplinarity select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; - -DROP TABLE IF EXISTS TARGET.result_new; \ No newline at end of file diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql deleted file mode 100644 index 478e3824e..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDBAll.sql +++ /dev/null @@ -1,150 +0,0 @@ -DROP TABLE IF EXISTS TARGET.result_new; - -create table TARGET.result_new as - select distinct * from ( - select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( - 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa - 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa - ) )) foo; - -INSERT INTO TARGET.result select * from TARGET.result_new; -ANALYZE TABLE TARGET.result_new COMPUTE STATISTICS; - -INSERT INTO TARGET.result select * from TARGET.result_new; -ANALYZE TABLE TARGET.result COMPUTE STATISTICS; - -INSERT INTO TARGET.result_citations select * from SOURCE.result_citations orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_citations COMPUTE STATISTICS; - -INSERT INTO TARGET.result_references_oc select * from SOURCE.result_references_oc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_references_oc COMPUTE STATISTICS; - -INSERT INTO TARGET.result_classifications select * from SOURCE.result_classifications orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_classifications COMPUTE STATISTICS; - -INSERT INTO TARGET.result_apc select * from SOURCE.result_apc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_apc COMPUTE STATISTICS; - -INSERT INTO TARGET.result_concepts select * from SOURCE.result_concepts orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_concepts COMPUTE STATISTICS; - -INSERT INTO TARGET.result_datasources select * from SOURCE.result_datasources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_datasources COMPUTE STATISTICS; - -INSERT INTO TARGET.result_fundercount select * from SOURCE.result_fundercount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fundercount COMPUTE STATISTICS; - -INSERT INTO TARGET.result_gold select * from SOURCE.result_gold orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_gold COMPUTE STATISTICS; - -INSERT INTO TARGET.result_greenoa select * from SOURCE.result_greenoa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_greenoa COMPUTE STATISTICS; - -INSERT INTO TARGET.result_languages select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_languages COMPUTE STATISTICS; - -INSERT INTO TARGET.result_licenses select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_licenses COMPUTE STATISTICS; - -INSERT INTO TARGET.result_oids select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_oids COMPUTE STATISTICS; - -INSERT INTO TARGET.result_organization select * from SOURCE.result_organization orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_organization COMPUTE STATISTICS; - -INSERT INTO TARGET.result_peerreviewed select * from SOURCE.result_peerreviewed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_peerreviewed COMPUTE STATISTICS; - -INSERT INTO TARGET.result_pids select * from SOURCE.result_pids orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_pids COMPUTE STATISTICS; - -INSERT INTO TARGET.result_projectcount select * from SOURCE.result_projectcount orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projectcount COMPUTE STATISTICS; - -INSERT INTO TARGET.result_projects select * from SOURCE.result_projects orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_projects COMPUTE STATISTICS; - -INSERT INTO TARGET.result_refereed select * from SOURCE.result_refereed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_refereed COMPUTE STATISTICS; - -INSERT INTO TARGET.result_sources select * from SOURCE.result_sources orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_sources COMPUTE STATISTICS; - -INSERT INTO TARGET.result_topics select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_topics COMPUTE STATISTICS; - -INSERT INTO TARGET.result_fos select * from SOURCE.result_fos orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_fos COMPUTE STATISTICS; - -INSERT INTO TARGET.result_accessroute select * from SOURCE.result_accessroute orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.result_accessroute COMPUTE STATISTICS; - -create or replace view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result_new); -create or replace view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result_new); -insert into TARGET.result_result select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou; -drop view TARGET.foo1; -drop view TARGET.foo2; -ANALYZE TABLE TARGET.result_result COMPUTE STATISTICS; - - --- indicators --- Sprint 1 ---- -INSERT INTO TARGET.indi_pub_green_oa select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_green_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_grey_lit select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_grey_lit COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_doi_from_crossref select * from SOURCE.indi_pub_doi_from_crossref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_doi_from_crossref COMPUTE STATISTICS; --- Sprint 2 ---- -INSERT INTO TARGET.indi_result_has_cc_licence select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_has_cc_licence_url select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_has_cc_licence_url COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_has_abstract select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_abstract COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_with_orcid select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_orcid COMPUTE STATISTICS; ----- Sprint 3 ---- -INSERT INTO TARGET.indi_funded_result_with_fundref select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_funded_result_with_fundref COMPUTE STATISTICS; - ----- Sprint 4 ---- -INSERT INTO TARGET.indi_pub_diamond select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_diamond COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_in_transformative select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_transformative COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_closed_other_open select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_closed_other_open COMPUTE STATISTICS; ----- Sprint 5 ---- -INSERT INTO TARGET.indi_result_no_of_copies select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_no_of_copies COMPUTE STATISTICS; ----- Sprint 6 ---- -INSERT INTO TARGET.indi_pub_hybrid_oa_with_cc select * from SOURCE.indi_pub_hybrid_oa_with_cc orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid_oa_with_cc COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_bronze_oa select * from SOURCE.indi_pub_bronze_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_bronze_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads select * from SOURCE.indi_pub_downloads orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_datasource select * from SOURCE.indi_pub_downloads_datasource orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_year select * from SOURCE.indi_pub_downloads_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_year COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_downloads_datasource_year select * from SOURCE.indi_pub_downloads_datasource_year orig where exists (select 1 from TARGET.result_new r where r.id=orig.result_id); -ANALYZE TABLE TARGET.indi_pub_downloads_datasource_year COMPUTE STATISTICS; ----- Sprint 7 ---- -INSERT INTO TARGET.indi_pub_gold_oa select * from SOURCE.indi_pub_gold_oa orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_gold_oa COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_hybrid select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_hybrid COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_has_preprint select * from SOURCE.indi_pub_has_preprint orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_has_preprint COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_in_subscribed select * from SOURCE.indi_pub_in_subscribed orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_in_subscribed COMPUTE STATISTICS; -INSERT INTO TARGET.indi_result_with_pid select * from SOURCE.indi_result_with_pid orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_result_with_pid COMPUTE STATISTICS; -INSERT INTO TARGET.indi_impact_measures select * from SOURCE.indi_impact_measures orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_impact_measures COMPUTE STATISTICS; -INSERT INTO TARGET.indi_pub_interdisciplinarity select * from SOURCE.indi_pub_interdisciplinarity orig where exists (select 1 from TARGET.result_new r where r.id=orig.id); -ANALYZE TABLE TARGET.indi_pub_interdisciplinarity COMPUTE STATISTICS; - -DROP TABLE IF EXISTS TARGET.result_new; diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql deleted file mode 100644 index 236f3733f..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/scripts/updateMonitorDB_institutions.sql +++ /dev/null @@ -1,12 +0,0 @@ -DROP TABLE IF EXISTS TARGET.result_new; - -create table TARGET.result_new as - select distinct * from ( - select * from SOURCE.result r where exists (select 1 from SOURCE.result_organization ro where ro.id=r.id and ro.organization in ( - 'openorgs____::4d4051b56708688235252f1d8fddb8c1', --Iscte - Instituto Universitário de Lisboa - 'openorgs____::ab4ac74c35fa5dada770cf08e5110fab' -- Universidade Católica Portuguesa - ) )) foo; - -INSERT INTO TARGET.result select * from TARGET.result_new; -ANALYZE TABLE TARGET.result_new COMPUTE STATISTICS; - diff --git a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml b/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml deleted file mode 100644 index 7b999a843..000000000 --- a/dhp-workflows/dhp-stats-monitor-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats-monitor/oozie_app/workflow.xml +++ /dev/null @@ -1,110 +0,0 @@ - - - - stats_db_name - the target stats database name - - - monitor_db_name - the target monitor db name - - - monitor_db_shadow_name - the name of the shadow monitor db - - - hive_metastore_uris - hive server metastore URIs - - - hive_jdbc_url - hive server jdbc url - - - hive_timeout - the time period, in seconds, after which Hive fails a transaction if a Hive client has not sent a hearbeat. The default value is 300 seconds. - - - hadoop_user_name - user name of the wf owner - - - - - ${jobTracker} - ${nameNode} - - - hive.metastore.uris - ${hive_metastore_uris} - - - hive.txn.timeout - ${hive_timeout} - - - mapred.job.queue.name - analytics - - - - - - - - ${wf:conf('resumeFrom') eq 'Step1-updateMonitorDB'} - ${wf:conf('resumeFrom') eq 'Step2-copyDataToImpalaCluster'} - ${wf:conf('resumeFrom') eq 'Step3-finalizeImpalaCluster'} - - - - - - Action failed, error message[${wf:errorMessage(wf:lastErrorNode())}] - - - - - ${jobTracker} - ${nameNode} - monitor.sh - ${stats_db_name} - ${monitor_db_name} - ${monitor_db_shadow_name} - ${wf:appPath()}/scripts/updateMonitorDB_institutions.sql - ${wf:appPath()}/scripts/updateMonitorDB.sql - ${wf:appPath()}/scripts/updateMonitorDBAll.sql - monitor.sh - - - - - - - - ${jobTracker} - ${nameNode} - copyDataToImpalaCluster.sh - ${monitor_db_name} - ${hadoop_user_name} - copyDataToImpalaCluster.sh - - - - - - - - ${jobTracker} - ${nameNode} - finalizeImpalaCluster.sh - ${monitor_db_name} - ${monitor_db_shadow_name} - finalizeImpalaCluster.sh - - - - - - - From a431b04814dc55c56ab2bde7d2a2663b7fc0950a Mon Sep 17 00:00:00 2001 From: "miriam.baglioni" Date: Tue, 10 Oct 2023 12:53:57 +0200 Subject: [PATCH 11/12] leftover for the properties and removal of bipfinder --- .../PrepareBipFinder.java | 178 ------------------ .../oozie_app/workflow.xml | 31 +-- .../createunresolvedentities/PrepareTest.java | 139 -------------- .../createunresolvedentities/ProduceTest.java | 30 --- 4 files changed, 1 insertion(+), 377 deletions(-) delete mode 100644 dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java diff --git a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java b/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java deleted file mode 100644 index 0507f90e5..000000000 --- a/dhp-workflows/dhp-aggregation/src/main/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareBipFinder.java +++ /dev/null @@ -1,178 +0,0 @@ - -package eu.dnetlib.dhp.actionmanager.createunresolvedentities; - -import static eu.dnetlib.dhp.actionmanager.Constants.*; -import static eu.dnetlib.dhp.actionmanager.Constants.UPDATE_CLASS_NAME; -import static eu.dnetlib.dhp.common.SparkSessionSupport.runWithSparkSession; - -import java.io.Serializable; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; -import java.util.stream.Collectors; - -import org.apache.commons.io.IOUtils; -import org.apache.spark.SparkConf; -import org.apache.spark.api.java.JavaRDD; -import org.apache.spark.api.java.JavaSparkContext; -import org.apache.spark.api.java.function.MapFunction; -import org.apache.spark.sql.Encoders; -import org.apache.spark.sql.SaveMode; -import org.apache.spark.sql.SparkSession; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import eu.dnetlib.dhp.actionmanager.bipmodel.BipScore; -import eu.dnetlib.dhp.actionmanager.bipmodel.score.deserializers.BipResultModel; -import eu.dnetlib.dhp.application.ArgumentApplicationParser; -import eu.dnetlib.dhp.common.HdfsSupport; -import eu.dnetlib.dhp.schema.common.ModelConstants; -import eu.dnetlib.dhp.schema.oaf.Instance; -import eu.dnetlib.dhp.schema.oaf.KeyValue; -import eu.dnetlib.dhp.schema.oaf.Measure; -import eu.dnetlib.dhp.schema.oaf.Result; -import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions; -import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils; -import eu.dnetlib.dhp.utils.DHPUtils; - -public class PrepareBipFinder implements Serializable { - - private static final Logger log = LoggerFactory.getLogger(PrepareBipFinder.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); - - public static void main(String[] args) throws Exception { - - String jsonConfiguration = IOUtils - .toString( - PrepareBipFinder.class - .getResourceAsStream( - "/eu/dnetlib/dhp/actionmanager/createunresolvedentities/prepare_parameters.json")); - - final ArgumentApplicationParser parser = new ArgumentApplicationParser(jsonConfiguration); - - parser.parseArgument(args); - - Boolean isSparkSessionManaged = Optional - .ofNullable(parser.get("isSparkSessionManaged")) - .map(Boolean::valueOf) - .orElse(Boolean.TRUE); - - log.info("isSparkSessionManaged: {}", isSparkSessionManaged); - - final String sourcePath = parser.get("sourcePath"); - log.info("sourcePath {}: ", sourcePath); - - final String outputPath = parser.get("outputPath"); - log.info("outputPath {}: ", outputPath); - - SparkConf conf = new SparkConf(); - - runWithSparkSession( - conf, - isSparkSessionManaged, - spark -> { - HdfsSupport.remove(outputPath, spark.sparkContext().hadoopConfiguration()); - prepareResults(spark, sourcePath, outputPath); - }); - } - - private static void prepareResults(SparkSession spark, String inputPath, String outputPath) { - - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - - JavaRDD bipDeserializeJavaRDD = sc - .textFile(inputPath) - .map(item -> OBJECT_MAPPER.readValue(item, BipResultModel.class)); - - spark - .createDataset(bipDeserializeJavaRDD.flatMap(entry -> entry.keySet().stream().map(key -> { - BipScore bs = new BipScore(); - bs.setId(key); - bs.setScoreList(entry.get(key)); - - return bs; - }).collect(Collectors.toList()).iterator()).rdd(), Encoders.bean(BipScore.class)) - .map((MapFunction) v -> { - Result r = new Result(); - final String cleanedPid = CleaningFunctions.normalizePidValue(DOI, v.getId()); - - r.setId(DHPUtils.generateUnresolvedIdentifier(v.getId(), DOI)); - Instance inst = new Instance(); - inst.setMeasures(getMeasure(v)); - - inst - .setPid( - Arrays - .asList( - OafMapperUtils - .structuredProperty( - cleanedPid, - OafMapperUtils - .qualifier( - DOI, DOI_CLASSNAME, - ModelConstants.DNET_PID_TYPES, - ModelConstants.DNET_PID_TYPES), - null))); - r.setInstance(Arrays.asList(inst)); - r - .setDataInfo( - OafMapperUtils - .dataInfo( - false, null, true, - false, - OafMapperUtils - .qualifier( - ModelConstants.PROVENANCE_ENRICH, - null, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - null)); - return r; - }, Encoders.bean(Result.class)) - .write() - .mode(SaveMode.Overwrite) - .option("compression", "gzip") - .json(outputPath + "/bip"); - } - - private static List getMeasure(BipScore value) { - return value - .getScoreList() - .stream() - .map(score -> { - Measure m = new Measure(); - m.setId(score.getId()); - m - .setUnit( - score - .getUnit() - .stream() - .map(unit -> { - KeyValue kv = new KeyValue(); - kv.setValue(unit.getValue()); - kv.setKey(unit.getKey()); - kv - .setDataInfo( - OafMapperUtils - .dataInfo( - false, - UPDATE_DATA_INFO_TYPE, - true, - false, - OafMapperUtils - .qualifier( - UPDATE_MEASURE_BIP_CLASS_ID, - UPDATE_CLASS_NAME, - ModelConstants.DNET_PROVENANCE_ACTIONS, - ModelConstants.DNET_PROVENANCE_ACTIONS), - "")); - return kv; - }) - .collect(Collectors.toList())); - return m; - }) - .collect(Collectors.toList()); - } -} diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml index a2935a71d..a5388f28b 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml @@ -5,11 +5,6 @@ fosPath the input path of the resources to be extended - - - - - outputPath the path where to store the actionset @@ -77,35 +72,10 @@ - - - - - - - - - - - - - - - - - - - - - - - - - yarn @@ -125,6 +95,7 @@ --sourcePath${fosPath} --outputPath${workingDir}/input/fos + --delimiter${delimiter} diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java index ccb0ebbff..da7bcd3de 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/PrepareTest.java @@ -67,92 +67,6 @@ public class PrepareTest { spark.stop(); } - @Test - void bipPrepareTest() throws Exception { - final String sourcePath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json") - .getPath(); - - PrepareBipFinder - .main( - new String[] { - "--isSparkSessionManaged", Boolean.FALSE.toString(), - "--sourcePath", sourcePath, - "--outputPath", workingDir.toString() + "/work" - - }); - - final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); - - JavaRDD tmp = sc - .textFile(workingDir.toString() + "/work/bip") - .map(item -> OBJECT_MAPPER.readValue(item, Result.class)); - - Assertions.assertEquals(86, tmp.count()); - - String doi1 = "unresolved::10.0000/096020199389707::doi"; - - Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi1)).count()); - Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi1)).collect().get(0).getInstance().size()); - Assertions - .assertEquals( - 3, tmp.filter(r -> r.getId().equals(doi1)).collect().get(0).getInstance().get(0).getMeasures().size()); - Assertions - .assertEquals( - "6.34596412687e-09", tmp - .filter(r -> r.getId().equals(doi1)) - .collect() - .get(0) - .getInstance() - .get(0) - .getMeasures() - .stream() - .filter(sl -> sl.getId().equals("influence")) - .collect(Collectors.toList()) - .get(0) - .getUnit() - .get(0) - .getValue()); - Assertions - .assertEquals( - "0.641151896994", tmp - .filter(r -> r.getId().equals(doi1)) - .collect() - .get(0) - .getInstance() - .get(0) - .getMeasures() - .stream() - .filter(sl -> sl.getId().equals("popularity_alt")) - .collect(Collectors.toList()) - .get(0) - .getUnit() - .get(0) - .getValue()); - Assertions - .assertEquals( - "2.33375102921e-09", tmp - .filter(r -> r.getId().equals(doi1)) - .collect() - .get(0) - .getInstance() - .get(0) - .getMeasures() - .stream() - .filter(sl -> sl.getId().equals("popularity")) - .collect(Collectors.toList()) - .get(0) - .getUnit() - .get(0) - .getValue()); - - final String doi2 = "unresolved::10.3390/s18072310::doi"; - - Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi2)).count()); - Assertions.assertEquals(1, tmp.filter(r -> r.getId().equals(doi2)).collect().get(0).getInstance().size()); - - } - @Test void fosPrepareTest() throws Exception { final String sourcePath = getClass() @@ -338,57 +252,4 @@ public class PrepareTest { } -// @Test -// void test3() throws Exception { -// final String sourcePath = "/Users/miriam.baglioni/Downloads/doi_fos_results_20_12_2021.csv.gz"; -// -// final String outputPath = workingDir.toString() + "/fos.json"; -// GetFOSSparkJob -// .main( -// new String[] { -// "--isSparkSessionManaged", Boolean.FALSE.toString(), -// "--sourcePath", sourcePath, -// -// "-outputPath", outputPath -// -// }); -// -// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); -// -// JavaRDD tmp = sc -// .textFile(outputPath) -// .map(item -> OBJECT_MAPPER.readValue(item, FOSDataModel.class)); -// -// tmp.foreach(t -> Assertions.assertTrue(t.getDoi() != null)); -// tmp.foreach(t -> Assertions.assertTrue(t.getLevel1() != null)); -// tmp.foreach(t -> Assertions.assertTrue(t.getLevel2() != null)); -// tmp.foreach(t -> Assertions.assertTrue(t.getLevel3() != null)); -// -// } -// -// @Test -// void test4() throws Exception { -// final String sourcePath = "/Users/miriam.baglioni/Downloads/doi_sdg_results_20_12_21.csv.gz"; -// -// final String outputPath = workingDir.toString() + "/sdg.json"; -// GetSDGSparkJob -// .main( -// new String[] { -// "--isSparkSessionManaged", Boolean.FALSE.toString(), -// "--sourcePath", sourcePath, -// -// "-outputPath", outputPath -// -// }); -// -// final JavaSparkContext sc = JavaSparkContext.fromSparkContext(spark.sparkContext()); -// -// JavaRDD tmp = sc -// .textFile(outputPath) -// .map(item -> OBJECT_MAPPER.readValue(item, SDGDataModel.class)); -// -// tmp.foreach(t -> Assertions.assertTrue(t.getDoi() != null)); -// tmp.foreach(t -> Assertions.assertTrue(t.getSbj() != null)); -// -// } } diff --git a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java index fce6c1e97..ce116688a 100644 --- a/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java +++ b/dhp-workflows/dhp-aggregation/src/test/java/eu/dnetlib/dhp/actionmanager/createunresolvedentities/ProduceTest.java @@ -340,18 +340,7 @@ public class ProduceTest { } private JavaRDD getResultJavaRDD() throws Exception { - final String bipPath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json") - .getPath(); - PrepareBipFinder - .main( - new String[] { - "--isSparkSessionManaged", Boolean.FALSE.toString(), - "--sourcePath", bipPath, - "--outputPath", workingDir.toString() + "/work" - - }); final String fosPath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos.json") .getPath(); @@ -449,18 +438,7 @@ public class ProduceTest { } private JavaRDD getResultJavaRDDPlusSDG() throws Exception { - final String bipPath = getClass() - .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/bip/bip.json") - .getPath(); - PrepareBipFinder - .main( - new String[] { - "--isSparkSessionManaged", Boolean.FALSE.toString(), - "--sourcePath", bipPath, - "--outputPath", workingDir.toString() + "/work" - - }); final String fosPath = getClass() .getResource("/eu/dnetlib/dhp/actionmanager/createunresolvedentities/fos/fos.json") .getPath(); @@ -517,14 +495,6 @@ public class ProduceTest { .filter(row -> row.getSubject() != null) .count()); - Assertions - .assertEquals( - 85, - tmp - .filter(row -> !row.getId().equals(doi)) - .filter(r -> r.getInstance() != null && r.getInstance().size() > 0) - .count()); - } @Test From a460ebe215ebe1f535905d0d3121a84bcd087c2b Mon Sep 17 00:00:00 2001 From: Claudio Atzori Date: Tue, 10 Oct 2023 15:50:11 +0200 Subject: [PATCH 12/12] [UnresolvedEntities] updated action name --- .../createunresolvedentities/oozie_app/workflow.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml index a5388f28b..c8e9547dc 100644 --- a/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-aggregation/src/main/resources/eu/dnetlib/dhp/actionmanager/createunresolvedentities/oozie_app/workflow.xml @@ -184,7 +184,7 @@ yarn cluster - Saves the result produced for bip and fos by grouping results with the same id + Save the unresolved entities grouping results with the same id eu.dnetlib.dhp.actionmanager.createunresolvedentities.SparkSaveUnresolved dhp-aggregation-${projectVersion}.jar