forked from D-Net/dnet-hadoop
moved stats computation in impala instead of hive
This commit is contained in:
parent
b8e96c8ae7
commit
33c85d4e66
|
|
@ -0,0 +1,17 @@
|
|||
export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
|
||||
export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
|
||||
if ! [ -L $link_folder ]
|
||||
then
|
||||
rm -Rf "$link_folder"
|
||||
ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
|
||||
fi
|
||||
|
||||
export SOURCE=$1
|
||||
export SHADOW=$2
|
||||
|
||||
echo "Updating shadow database"
|
||||
impala-shell -q "invalidate metadata ${SOURCE}"
|
||||
impala-shell -q "create database if not exists ${SHADOW}"
|
||||
impala-shell -d ${SHADOW} -q "show tables" --delimited | sed "s/^/drop view if exists ${SHADOW}./" | sed "s/$/;/" | impala-shell -f -
|
||||
impala-shell -d ${SOURCE} -q "show tables" --delimited | sed "s/\(.*\)/create view ${SHADOW}.\1 as select * from ${SOURCE}.\1;/" | impala-shell -f -
|
||||
echo "Shadow db ready!"
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
|
||||
export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
|
||||
if ! [ -L $link_folder ]
|
||||
then
|
||||
rm -Rf "$link_folder"
|
||||
ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
|
||||
fi
|
||||
|
||||
echo "Getting file from " $3
|
||||
hdfs dfs -copyToLocal $3
|
||||
|
||||
echo "Running impala shell make the new database visible"
|
||||
impala-shell -q "INVALIDATE METADATA;"
|
||||
|
||||
echo "Running impala shell to compute new table stats"
|
||||
impala-shell -d $1 -f $2
|
||||
echo "Impala shell finished"
|
||||
rm $2
|
||||
|
|
@ -43,6 +43,6 @@ FROM ${external_stats_db_name}.concept;
|
|||
------------------------------------------------------------------------------------------------
|
||||
create table ${stats_db_name}.creation_date as
|
||||
select date_format(current_date(), 'dd-MM-yyyy') as date;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.creation_date COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.creation_date COMPUTE STATISTICS FOR COLUMNS;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.creation_date COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.creation_date COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -114,5 +114,5 @@ WHERE result_projects.id = result.id
|
|||
AND result.type = 'publication'
|
||||
AND project.id = result_projects.project;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.project COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.project COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -65,15 +65,15 @@ WHERE r.id = rd.id
|
|||
AND r.bestlicence = 'Open Access'
|
||||
GROUP BY org.country;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -57,12 +57,12 @@ UNION ALL
|
|||
SELECT * FROM ${stats_db_name}.software_sources
|
||||
UNION ALL
|
||||
SELECT * FROM ${stats_db_name}.otherresearchproduct_sources;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.publication_sources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_sources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_sources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_sources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -48,15 +48,15 @@ FROM (
|
|||
from ${openaire_db_name}.datasource d
|
||||
WHERE d.datainfo.deletedbyinference=false) d on o.datasource = d.id;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.publication_licenses COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_licenses COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_licenses COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_licenses COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_licenses COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_licenses COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_licenses COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_licenses COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.organization_pids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.organization_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.organization_sources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.organization_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_licenses COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_licenses COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_licenses COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_licenses COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_licenses COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_licenses COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_licenses COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_licenses COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.organization_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.organization_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.organization_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.organization_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -34,12 +34,12 @@ union all
|
|||
select * from ${stats_db_name}.software_refereed
|
||||
union all
|
||||
select * from ${stats_db_name}.otherresearchproduct_refereed;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.publication_refereed COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_refereed COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_refereed COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_refereed COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_refereed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_refereed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_refereed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_refereed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -79,13 +79,13 @@ join ${stats_db_name}.datasource_organizations dor on dor.id=d.id
|
|||
join ${stats_db_name}.organization o on o.id=dor.organization
|
||||
where o.country is not null and o.country!='';
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.result_peerreviewed COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_peerreviewed COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_greenoa COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_greenoa COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_gold COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_gold COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_affiliated_country COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_affiliated_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_deposited_country COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_deposited_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_peerreviewed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_peerreviewed COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_greenoa COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_greenoa COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_gold COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_gold COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_affiliated_country COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_affiliated_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_deposited_country COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_deposited_country COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -53,6 +53,6 @@ drop table if exists ${stats_db_name}.result;
|
|||
drop view if exists ${stats_db_name}.result;
|
||||
create table ${stats_db_name}.result stored as parquet as select * from ${stats_db_name}.result_tmp;
|
||||
drop table ${stats_db_name}.result_tmp;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.result COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result COMPUTE STATISTICS FOR COLUMNS;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.result COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -30,10 +30,10 @@ from rcount
|
|||
group by rcount.pid;
|
||||
|
||||
create view ${stats_db_name}.rndexpenditure as select * from stats_ext.rndexpediture;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.result_projectcount COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_projectcount COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_fundercount COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_fundercount COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_resultcount COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_resultcount COMPUTE STATISTICS FOR COLUMNS;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_projectcount COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_projectcount COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_fundercount COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_fundercount COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_resultcount COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_resultcount COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -1,207 +0,0 @@
|
|||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
-- Shadow schema table exchange
|
||||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
|
||||
-- Dropping old views
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.category;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.concept;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.context;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.country;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.countrygdp;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.creation_date;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_classifications;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_concepts;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_datasources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_languages;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_topics;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_languages;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_organizations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_results;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.funder;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.fundref;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.numbers_country;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_datasources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_projects;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_classifications;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_concepts;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_datasources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_languages;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_topics;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_organizations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_results;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_resultcount;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_results_publication;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_classifications;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_concepts;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_datasources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_languages;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_topics;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_affiliated_country;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_classifications;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_concepts;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_datasources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_deposited_country;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_fundercount;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_gold;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_greenoa;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_languages;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_organization;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_peerreviewed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_projectcount;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_projects;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_topics;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.rndexpediture;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.roarmap;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_citations;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_classifications;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_concepts;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_datasources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_languages;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_licenses;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_oids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_pids;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_refereed;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_sources;
|
||||
DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_topics;
|
||||
|
||||
|
||||
-- Creating the shadow database, in case it doesn't exist
|
||||
CREATE database IF NOT EXISTS ${stats_db_shadow_name};
|
||||
|
||||
-- Creating new views
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.category AS SELECT * FROM ${stats_db_name}.category;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.concept AS SELECT * FROM ${stats_db_name}.concept;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.context AS SELECT * FROM ${stats_db_name}.context;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.country AS SELECT * FROM ${stats_db_name}.country;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.countrygdp AS SELECT * FROM ${stats_db_name}.countrygdp;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.creation_date AS SELECT * FROM ${stats_db_name}.creation_date;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset AS SELECT * FROM ${stats_db_name}.dataset;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_citations AS SELECT * FROM ${stats_db_name}.dataset_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_classifications AS SELECT * FROM ${stats_db_name}.dataset_classifications;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_concepts AS SELECT * FROM ${stats_db_name}.dataset_concepts;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_datasources AS SELECT * FROM ${stats_db_name}.dataset_datasources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_languages AS SELECT * FROM ${stats_db_name}.dataset_languages;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_licenses AS SELECT * FROM ${stats_db_name}.dataset_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_oids AS SELECT * FROM ${stats_db_name}.dataset_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_pids AS SELECT * FROM ${stats_db_name}.dataset_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_refereed AS SELECT * FROM ${stats_db_name}.dataset_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_sources AS SELECT * FROM ${stats_db_name}.dataset_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_topics AS SELECT * FROM ${stats_db_name}.dataset_topics;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource AS SELECT * FROM ${stats_db_name}.datasource;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_languages AS SELECT * FROM ${stats_db_name}.datasource_languages;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_oids AS SELECT * FROM ${stats_db_name}.datasource_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_organizations AS SELECT * FROM ${stats_db_name}.datasource_organizations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_results AS SELECT * FROM ${stats_db_name}.datasource_results;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_sources AS SELECT * FROM ${stats_db_name}.datasource_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.funder AS SELECT * FROM ${stats_db_name}.funder;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.fundref AS SELECT * FROM ${stats_db_name}.fundref;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.numbers_country AS SELECT * FROM ${stats_db_name}.numbers_country;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization AS SELECT * FROM ${stats_db_name}.organization;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_datasources AS SELECT * FROM ${stats_db_name}.organization_datasources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_pids AS SELECT * FROM ${stats_db_name}.organization_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_projects AS SELECT * FROM ${stats_db_name}.organization_projects;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_sources AS SELECT * FROM ${stats_db_name}.organization_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct AS SELECT * FROM ${stats_db_name}.otherresearchproduct;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_citations AS SELECT * FROM ${stats_db_name}.otherresearchproduct_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_classifications AS SELECT * FROM ${stats_db_name}.otherresearchproduct_classifications;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_concepts AS SELECT * FROM ${stats_db_name}.otherresearchproduct_concepts;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_datasources AS SELECT * FROM ${stats_db_name}.otherresearchproduct_datasources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_languages AS SELECT * FROM ${stats_db_name}.otherresearchproduct_languages;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_licenses AS SELECT * FROM ${stats_db_name}.otherresearchproduct_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_oids AS SELECT * FROM ${stats_db_name}.otherresearchproduct_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_pids AS SELECT * FROM ${stats_db_name}.otherresearchproduct_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_refereed AS SELECT * FROM ${stats_db_name}.otherresearchproduct_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_sources AS SELECT * FROM ${stats_db_name}.otherresearchproduct_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_topics AS SELECT * FROM ${stats_db_name}.otherresearchproduct_topics;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project AS SELECT * FROM ${stats_db_name}.project;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_oids AS SELECT * FROM ${stats_db_name}.project_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_organizations AS SELECT * FROM ${stats_db_name}.project_organizations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_results AS SELECT * FROM ${stats_db_name}.project_results;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_resultcount AS SELECT * FROM ${stats_db_name}.project_resultcount;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_results_publication AS SELECT * FROM ${stats_db_name}.project_results_publication;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication AS SELECT * FROM ${stats_db_name}.publication;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_citations AS SELECT * FROM ${stats_db_name}.publication_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_classifications AS SELECT * FROM ${stats_db_name}.publication_classifications;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_concepts AS SELECT * FROM ${stats_db_name}.publication_concepts;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_datasources AS SELECT * FROM ${stats_db_name}.publication_datasources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_languages AS SELECT * FROM ${stats_db_name}.publication_languages;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_licenses AS SELECT * FROM ${stats_db_name}.publication_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_oids AS SELECT * FROM ${stats_db_name}.publication_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_pids AS SELECT * FROM ${stats_db_name}.publication_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_refereed AS SELECT * FROM ${stats_db_name}.publication_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_sources AS SELECT * FROM ${stats_db_name}.publication_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_topics AS SELECT * FROM ${stats_db_name}.publication_topics;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result AS SELECT * FROM ${stats_db_name}.result;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_affiliated_country AS SELECT * FROM ${stats_db_name}.result_affiliated_country;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_citations AS SELECT * FROM ${stats_db_name}.result_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_classifications AS SELECT * FROM ${stats_db_name}.result_classifications;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_concepts AS SELECT * FROM ${stats_db_name}.result_concepts;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_datasources AS SELECT * FROM ${stats_db_name}.result_datasources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_deposited_country AS SELECT * FROM ${stats_db_name}.result_deposited_country;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_fundercount AS SELECT * FROM ${stats_db_name}.result_fundercount;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_gold AS SELECT * FROM ${stats_db_name}.result_gold;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_greenoa AS SELECT * FROM ${stats_db_name}.result_greenoa;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_languages AS SELECT * FROM ${stats_db_name}.result_languages;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_licenses AS SELECT * FROM ${stats_db_name}.result_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_oids AS SELECT * FROM ${stats_db_name}.result_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_organization AS SELECT * FROM ${stats_db_name}.result_organization;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_peerreviewed AS SELECT * FROM ${stats_db_name}.result_peerreviewed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_pids AS SELECT * FROM ${stats_db_name}.result_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_projectcount AS SELECT * FROM ${stats_db_name}.result_projectcount;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_projects AS SELECT * FROM ${stats_db_name}.result_projects;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_refereed AS SELECT * FROM ${stats_db_name}.result_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_sources AS SELECT * FROM ${stats_db_name}.result_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_topics AS SELECT * FROM ${stats_db_name}.result_topics;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.rndexpediture AS SELECT * FROM ${stats_db_name}.rndexpediture;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.roarmap AS SELECT * FROM ${stats_db_name}.roarmap;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software AS SELECT * FROM ${stats_db_name}.software;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_citations AS SELECT * FROM ${stats_db_name}.software_citations;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_classifications AS SELECT * FROM ${stats_db_name}.software_classifications;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_concepts AS SELECT * FROM ${stats_db_name}.software_concepts;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_datasources AS SELECT * FROM ${stats_db_name}.software_datasources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_languages AS SELECT * FROM ${stats_db_name}.software_languages;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_licenses AS SELECT * FROM ${stats_db_name}.software_licenses;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_oids AS SELECT * FROM ${stats_db_name}.software_oids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_pids AS SELECT * FROM ${stats_db_name}.software_pids;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_refereed AS SELECT * FROM ${stats_db_name}.software_refereed;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_sources AS SELECT * FROM ${stats_db_name}.software_sources;
|
||||
CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_topics AS SELECT * FROM ${stats_db_name}.software_topics;
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
-- Impala table statistics - Needed to make the tables
|
||||
-- visible for impala
|
||||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
|
||||
INVALIDATE METADATA ${stats_db_name};
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
-- Impala table statistics - Needed to make the tables
|
||||
-- visible for impala
|
||||
------------------------------------------------------
|
||||
------------------------------------------------------
|
||||
|
||||
INVALIDATE METADATA ${stats_db_name};
|
||||
|
|
@ -93,21 +93,21 @@ FROM ${openaire_db_name}.publication p
|
|||
WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
|
||||
and p.datainfo.deletedbyinference = false;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.publication_tmp COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_classifications COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_concepts COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_datasources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_languages COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_oids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_pids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_topics COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_citations COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.publication_citations COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_classifications COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_concepts COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_datasources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_topics COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_citations COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_citations COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -93,20 +93,20 @@ SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subj
|
|||
FROM ${openaire_db_name}.dataset p
|
||||
LATERAL VIEW explode(p.subject) subjects AS subject
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_tmp COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_classifications COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_concepts COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_datasources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_languages COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_oids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_pids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_topics COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.dataset_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_classifications COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_concepts COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_datasources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_topics COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -93,20 +93,20 @@ SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subj
|
|||
FROM ${openaire_db_name}.software p
|
||||
LATERAL VIEW explode(p.subject) subjects AS subject
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.software_tmp COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_classifications COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_concepts COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_datasources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_languages COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_oids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_pids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_topics COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.software_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_classifications COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_concepts COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_datasources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_topics COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -85,19 +85,19 @@ SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subj
|
|||
FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_tmp COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_classifications COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_concepts COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_datasources COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_languages COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_oids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_pids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_topics COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.otherresearchproduct_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_classifications COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_concepts COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_datasources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_topics COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -72,13 +72,13 @@ select distinct xpath_string(fund, '//funder/id') as id,
|
|||
xpath_string(fund, '//funder/shortname') as shortname
|
||||
from ${openaire_db_name}.project p lateral view explode(p.fundingtree.value) fundingtree as fund;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.project_oids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_organizations COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_organizations COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_results COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_results COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_tmp COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.project_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.funder COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.funder COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_organizations COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_organizations COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_results COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_results COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.project_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.funder COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.funder COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -135,7 +135,7 @@ FROM ${stats_db_name}.result r
|
|||
JOIN ${stats_db_name}.project_results pr ON r.id = pr.result
|
||||
JOIN ${stats_db_name}.project_tmp p ON p.id = pr.id;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.result_organization COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_organization COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_projects COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.result_projects COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_organization COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_organization COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_projects COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_projects COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -99,11 +99,11 @@ CREATE OR REPLACE VIEW ${stats_db_name}.datasource_results AS
|
|||
SELECT datasource AS id, id AS result
|
||||
FROM ${stats_db_name}.result_datasources;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.datasource_tmp COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.datasource_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.datasource_languages COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.datasource_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.datasource_oids COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.datasource_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
ANALYZE TABLE ${stats_db_name}.datasource_organizations COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.datasource_organizations COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_organizations COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_organizations COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -19,5 +19,5 @@ CREATE OR REPLACE VIEW ${stats_db_name}.organization_projects AS
|
|||
SELECT id AS project, organization as id
|
||||
FROM ${stats_db_name}.project_organizations;
|
||||
|
||||
ANALYZE TABLE ${stats_db_name}.organization COMPUTE STATISTICS;
|
||||
ANALYZE TABLE ${stats_db_name}.organization COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.organization COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.organization COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
|
@ -260,31 +260,6 @@
|
|||
<param>stats_db_name=${stats_db_name}</param>
|
||||
<param>openaire_db_name=${openaire_db_name}</param>
|
||||
</hive2>
|
||||
<ok to="Step17"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="Step17">
|
||||
<hive2 xmlns="uri:oozie:hive2-action:0.1">
|
||||
<jdbc-url>${hive_jdbc_url}</jdbc-url>
|
||||
<script>scripts/step17.sql</script>
|
||||
<param>stats_db_name=${stats_db_name}</param>
|
||||
<param>stats_db_shadow_name=${stats_db_shadow_name}</param>
|
||||
</hive2>
|
||||
<ok to="Step18"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
||||
<action name="Step18">
|
||||
<shell xmlns="uri:oozie:shell-action:0.1">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<exec>impala-shell.sh</exec>
|
||||
<argument>${stats_db_name}</argument>
|
||||
<argument>step18.sql</argument>
|
||||
<argument>${wf:appPath()}/scripts/step18.sql</argument>
|
||||
<file>impala-shell.sh</file>
|
||||
</shell>
|
||||
<ok to="Step19"/>
|
||||
<error to="Kill"/>
|
||||
</action>
|
||||
|
|
@ -293,11 +268,10 @@
|
|||
<shell xmlns="uri:oozie:shell-action:0.1">
|
||||
<job-tracker>${jobTracker}</job-tracker>
|
||||
<name-node>${nameNode}</name-node>
|
||||
<exec>impala-shell.sh</exec>
|
||||
<exec>finalizedb.sh</exec>
|
||||
<argument>stats_db_name</argument>
|
||||
<argument>${stats_db_shadow_name}</argument>
|
||||
<argument>step19.sql</argument>
|
||||
<argument>${wf:appPath()}/scripts/step19.sql</argument>
|
||||
<file>impala-shell.sh</file>
|
||||
<file>finalizedb.sh</file>
|
||||
</shell>
|
||||
<ok to="step20-createMonitorDB"/>
|
||||
<error to="Kill"/>
|
||||
|
|
|
|||
Loading…
Reference in New Issue