forked from D-Net/dnet-hadoop
Merge pull request '[beta] stats update workflow' (#128) from antonis.lempesis/dnet-hadoop:beta into beta
Reviewed-on: D-Net/dnet-hadoop#128
This commit is contained in:
commit
dc55ed4acd
|
|
@ -13,7 +13,7 @@ echo "Getting file from " $SCRIPT_PATH
|
|||
hdfs dfs -copyToLocal $SCRIPT_PATH
|
||||
|
||||
echo "Creating indicators"
|
||||
impala-shell -d ${TARGET} -q "invalidate metadata"
|
||||
impala-shell -q "invalidate metadata"
|
||||
impala-shell -d ${TARGET} -q "show tables" --delimited | sed "s/^\(.*\)/compute stats ${TARGET}.\1;/" | impala-shell -c -f -
|
||||
cat step16_7-createIndicatorsTables.sql | impala-shell -d $TARGET -f -
|
||||
echo "Indicators created"
|
||||
|
|
@ -57,12 +57,14 @@ UNION ALL
|
|||
SELECT * FROM ${stats_db_name}.software_sources
|
||||
UNION ALL
|
||||
SELECT * FROM ${stats_db_name}.otherresearchproduct_sources;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_sources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_sources COMPUTE STATISTICS FOR COLUMNS;
|
||||
|
||||
|
||||
create table ${stats_db_name}.result_orcid as
|
||||
select distinct res.id, regexp_replace(res.orcid, 'http://orcid.org/' ,'') as orcid
|
||||
from (
|
||||
SELECT substr(res.id, 4) as id, auth_pid.value as orcid
|
||||
FROM ${openaire_db_name}.result res
|
||||
LATERAL VIEW explode(author) a as auth
|
||||
LATERAL VIEW explode(auth.pid) ap as auth_pid
|
||||
LATERAL VIEW explode(auth.pid.qualifier.classid) apt as author_pid_type
|
||||
WHERE res.datainfo.deletedbyinference = FALSE and res.datainfo.invisible = FALSE and author_pid_type = 'orcid') as res
|
||||
|
|
@ -33,13 +33,4 @@ select * from ${stats_db_name}.dataset_refereed
|
|||
union all
|
||||
select * from ${stats_db_name}.software_refereed
|
||||
union all
|
||||
select * from ${stats_db_name}.otherresearchproduct_refereed;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_refereed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_refereed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_refereed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_refereed COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_refereed COMPUTE STATISTICS FOR COLUMNS;
|
||||
select * from ${stats_db_name}.otherresearchproduct_refereed;
|
||||
|
|
@ -39,4 +39,198 @@ from publication p
|
|||
join result_instance ri on ri.id = p.id
|
||||
join datasource on datasource.id = ri.hostedby
|
||||
where datasource.id like '%doajarticles%') tmp
|
||||
on p.id= tmp.id;
|
||||
on p.id= tmp.id;
|
||||
|
||||
create table indi_project_pubs_count stored as parquet as
|
||||
select pr.id id, count(p.id) total_pubs from project_results pr
|
||||
join publication p on p.id=pr.result
|
||||
group by pr.id;
|
||||
|
||||
create table indi_project_datasets_count stored as parquet as
|
||||
select pr.id id, count(d.id) total_datasets from project_results pr
|
||||
join dataset d on d.id=pr.result
|
||||
group by pr.id;
|
||||
|
||||
create table indi_project_software_count stored as parquet as
|
||||
select pr.id id, count(s.id) total_software from project_results pr
|
||||
join software s on s.id=pr.result
|
||||
group by pr.id;
|
||||
|
||||
create table indi_project_otherresearch_count stored as parquet as
|
||||
select pr.id id, count(o.id) total_other from project_results pr
|
||||
join otherresearchproduct o on o.id=pr.result
|
||||
group by pr.id;
|
||||
|
||||
create table indi_pub_avg_year_country_oa stored as parquet as
|
||||
select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
|
||||
round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
|
||||
from
|
||||
(SELECT year, country, SUM(CASE
|
||||
WHEN bestlicence='Open Access' THEN 1
|
||||
ELSE 0
|
||||
END) AS OpenAccess, SUM(CASE
|
||||
WHEN bestlicence<>'Open Access' THEN 1
|
||||
ELSE 0
|
||||
END) AS NonOpenAccess
|
||||
FROM publication p
|
||||
join result_organization ro on p.id=ro.id
|
||||
join organization o on o.id=ro.organization
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by year, country) tmp;
|
||||
|
||||
create table indi_dataset_avg_year_country_oa stored as parquet as
|
||||
select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
|
||||
round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
|
||||
from
|
||||
(SELECT year, country, SUM(CASE
|
||||
WHEN bestlicence='Open Access' THEN 1
|
||||
ELSE 0
|
||||
END) AS OpenAccess, SUM(CASE
|
||||
WHEN bestlicence<>'Open Access' THEN 1
|
||||
ELSE 0
|
||||
END) AS NonOpenAccess
|
||||
FROM dataset d
|
||||
join result_organization ro on d.id=ro.id
|
||||
join organization o on o.id=ro.organization
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by year, country) tmp;
|
||||
|
||||
create table indi_software_avg_year_country_oa stored as parquet as
|
||||
select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
|
||||
round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
|
||||
from
|
||||
(SELECT year, country, SUM(CASE
|
||||
WHEN bestlicence='Open Access' THEN 1
|
||||
ELSE 0
|
||||
END) AS OpenAccess, SUM(CASE
|
||||
WHEN bestlicence<>'Open Access' THEN 1
|
||||
ELSE 0
|
||||
END) AS NonOpenAccess
|
||||
FROM software s
|
||||
join result_organization ro on s.id=ro.id
|
||||
join SOURCER.organization o on o.id=ro.organization
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by year, country) tmp;
|
||||
|
||||
|
||||
create table indi_other_avg_year_country_oa stored as parquet as
|
||||
select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
|
||||
round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
|
||||
from
|
||||
(SELECT year, country, SUM(CASE
|
||||
WHEN bestlicence='Open Access' THEN 1
|
||||
ELSE 0
|
||||
END) AS OpenAccess, SUM(CASE
|
||||
WHEN bestlicence<>'Open Access' THEN 1
|
||||
ELSE 0
|
||||
END) AS NonOpenAccess
|
||||
FROM otherresearchproduct orp
|
||||
join result_organization ro on orp.id=ro.id
|
||||
join organization o on o.id=ro.organization
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by year, country) tmp;
|
||||
|
||||
create table indi_pub_avg_year_context_oa stored as parquet as
|
||||
with total as
|
||||
(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from publication_concepts pc
|
||||
join context c on pc.concept like concat('%',c.id,'%')
|
||||
join publication p on p.id=pc.id
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by c.name, year )
|
||||
select year, name, round(no_of_pubs/total*100,3) averageofpubs
|
||||
from total;
|
||||
|
||||
create table indi_dataset_avg_year_context_oa stored as parquet as
|
||||
with total as
|
||||
(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from dataset_concepts pc
|
||||
join context c on pc.concept like concat('%',c.id,'%')
|
||||
join dataset p on p.id=pc.id
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by c.name, year )
|
||||
select year, name, round(no_of_pubs/total*100,3) averageofdataset
|
||||
from total;
|
||||
|
||||
create table indi_software_avg_year_context_oa stored as parquet as
|
||||
with total as
|
||||
(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from software_concepts pc
|
||||
join context c on pc.concept like concat('%',c.id,'%')
|
||||
join software p on p.id=pc.id
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by c.name, year )
|
||||
select year, name, round(no_of_pubs/total*100,3) averageofsoftware
|
||||
from total;
|
||||
|
||||
create table indi_other_avg_year_context_oa stored as parquet as
|
||||
with total as
|
||||
(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from otherresearchproduct_concepts pc
|
||||
join context c on pc.concept like concat('%',c.id,'%')
|
||||
join otherresearchproduct p on p.id=pc.id
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by c.name, year )
|
||||
select year, name, round(no_of_pubs/total*100,3) averageofother
|
||||
from total;
|
||||
|
||||
create table indi_other_avg_year_content_oa stored as parquet as
|
||||
with total as
|
||||
(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
|
||||
from otherresearchproduct_datasources pd
|
||||
join datasource d on datasource=d.id
|
||||
join otherresearchproduct p on p.id=pd.id
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by d.type, year)
|
||||
select year, type, round(no_of_pubs/total*100,3) averageOfOtherresearchproduct
|
||||
from total;
|
||||
|
||||
create table indi_software_avg_year_content_oa stored as parquet as
|
||||
with total as
|
||||
(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
|
||||
from software_datasources pd
|
||||
join datasource d on datasource=d.id
|
||||
join software p on p.id=pd.id
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by d.type, year)
|
||||
select year, type, round(no_of_pubs/total*100,3) averageOfSoftware
|
||||
from total;
|
||||
|
||||
create table indi_dataset_avg_year_content_oa stored as parquet as
|
||||
with total as
|
||||
(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
|
||||
from dataset_datasources pd
|
||||
join datasource d on datasource=d.id
|
||||
join dataset p on p.id=pd.id
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by d.type, year)
|
||||
select year, type, round(no_of_pubs/total*100,3) averageOfDatasets
|
||||
from total;
|
||||
|
||||
create table indi_pub_avg_year_content_oa stored as parquet as
|
||||
with total as
|
||||
(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
|
||||
from publication_datasources pd
|
||||
join datasource d on datasource=d.id
|
||||
join publication p on p.id=pd.id
|
||||
where cast(year as int)>=2003 and cast(year as int)<=2021
|
||||
group by d.type, year)
|
||||
select year, type, round(no_of_pubs/total*100,3) averageOfPubs
|
||||
from total;
|
||||
|
||||
create table indi_pub_has_cc_licence stored as parquet as
|
||||
select distinct p.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license
|
||||
from publication p
|
||||
left outer join (select p.id, license.type as lic from publication p
|
||||
join publication_licenses as license on license.id = p.id
|
||||
where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp
|
||||
on p.id= tmp.id;
|
||||
|
||||
create table indi_pub_has_cc_licence_url stored as parquet as
|
||||
select distinct p.id, (case when lic_host='' or lic_host is null then 0 else 1 end) as has_cc_license_url
|
||||
from publication p
|
||||
left outer join (select p.id, lower(parse_url(license.type, "HOST")) as lic_host
|
||||
from publication p
|
||||
join publication_licenses as license on license.id = p.id
|
||||
WHERE lower(parse_url(license.type, 'HOST')) = 'creativecommons.org') tmp
|
||||
on p.id= tmp.id;
|
||||
|
||||
create table indi_pub_has_abstract stored as parquet as
|
||||
select distinct publication.id, coalesce(abstract, 1) has_abstract
|
||||
from publication;
|
||||
|
|
@ -90,27 +90,8 @@ FROM ${openaire_db_name}.publication p
|
|||
where p.datainfo.deletedbyinference = false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.publication_citations AS
|
||||
SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result
|
||||
SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
|
||||
FROM ${openaire_db_name}.publication p
|
||||
lateral view explode(p.extrainfo) citations AS citation
|
||||
WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
|
||||
and p.datainfo.deletedbyinference = false;
|
||||
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_classifications COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_concepts COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_datasources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_topics COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_citations COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.publication_citations COMPUTE STATISTICS FOR COLUMNS;
|
||||
and p.datainfo.deletedbyinference = false;
|
||||
|
|
@ -41,7 +41,7 @@ FROM ${openaire_db_name}.dataset d
|
|||
WHERE d.datainfo.deletedbyinference = FALSE;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.dataset_citations AS
|
||||
SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result
|
||||
SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
|
||||
FROM ${openaire_db_name}.dataset d
|
||||
LATERAL VIEW explode(d.extrainfo) citations AS citation
|
||||
WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
|
||||
|
|
@ -95,21 +95,4 @@ CREATE TABLE ${stats_db_name}.dataset_topics AS
|
|||
SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
|
||||
FROM ${openaire_db_name}.dataset p
|
||||
LATERAL VIEW explode(p.subject) subjects AS subject
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_classifications COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_concepts COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_datasources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_topics COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.dataset_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
|
|
@ -41,7 +41,7 @@ from ${openaire_db_name}.software s
|
|||
where s.datainfo.deletedbyinference = false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.software_citations AS
|
||||
SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT
|
||||
SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
|
||||
FROM ${openaire_db_name}.software s
|
||||
LATERAL VIEW explode(s.extrainfo) citations as citation
|
||||
where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
|
||||
|
|
@ -95,21 +95,4 @@ CREATE TABLE ${stats_db_name}.software_topics AS
|
|||
SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
|
||||
FROM ${openaire_db_name}.software p
|
||||
LATERAL VIEW explode(p.subject) subjects AS subject
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
--
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_classifications COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_concepts COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_datasources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_topics COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.software_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
|
|
@ -41,7 +41,7 @@ WHERE o.datainfo.deletedbyinference = FALSE;
|
|||
|
||||
-- Otherresearchproduct_citations
|
||||
CREATE TABLE ${stats_db_name}.otherresearchproduct_citations AS
|
||||
SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT
|
||||
SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
|
||||
FROM ${openaire_db_name}.otherresearchproduct o LATERAL VIEW explode(o.extrainfo) citations AS citation
|
||||
WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
|
||||
and o.datainfo.deletedbyinference = false;
|
||||
|
|
@ -86,21 +86,4 @@ where p.datainfo.deletedbyinference = false;
|
|||
CREATE TABLE ${stats_db_name}.otherresearchproduct_topics AS
|
||||
SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
|
||||
FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_classifications COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_classifications COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_concepts COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_concepts COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_datasources COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_datasources COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_pids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_pids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_topics COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_topics COMPUTE STATISTICS FOR COLUMNS;
|
||||
where p.datainfo.deletedbyinference = false;
|
||||
|
|
@ -13,11 +13,17 @@ WHERE r.reltype = 'projectOrganization'
|
|||
and r.datainfo.deletedbyinference = false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.project_results AS
|
||||
SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS result
|
||||
SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS result, r.datainfo.provenanceaction.classname as provenance
|
||||
FROM ${openaire_db_name}.relation r
|
||||
WHERE r.reltype = 'resultProject'
|
||||
and r.datainfo.deletedbyinference = false;
|
||||
|
||||
create table ${stats_db_name}.project_classification as
|
||||
select substr(p.id, 4) as id, class.h2020programme.code, class.level1, class.level2, class.level3
|
||||
from ${openaire_db_name}.project p
|
||||
lateral view explode(p.h2020classification) classifs as class
|
||||
where p.datainfo.deletedbyinference=false and class.h2020programme is not null;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.project_tmp
|
||||
(
|
||||
id STRING,
|
||||
|
|
|
|||
|
|
@ -130,12 +130,7 @@ WHERE r.reltype = 'resultOrganization'
|
|||
and r.datainfo.deletedbyinference = false;
|
||||
|
||||
CREATE TABLE ${stats_db_name}.result_projects AS
|
||||
select pr.result AS id, pr.id AS project, datediff(p.enddate, p.startdate) AS daysfromend
|
||||
select pr.result AS id, pr.id AS project, datediff(p.enddate, p.startdate) AS daysfromend, pr.provenance as provenance
|
||||
FROM ${stats_db_name}.result r
|
||||
JOIN ${stats_db_name}.project_results pr ON r.id = pr.result
|
||||
JOIN ${stats_db_name}.project_tmp p ON p.id = pr.id;
|
||||
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_organization COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_organization COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_projects COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.result_projects COMPUTE STATISTICS FOR COLUMNS;
|
||||
JOIN ${stats_db_name}.project_tmp p ON p.id = pr.id;
|
||||
|
|
@ -17,7 +17,9 @@ CREATE TABLE ${stats_db_name}.datasource_tmp
|
|||
`latitude` STRING,
|
||||
`longitude` STRING,
|
||||
`websiteurl` STRING,
|
||||
`compatibility` STRING
|
||||
`compatibility` STRING,
|
||||
issn_printed STRING,
|
||||
issn_online STRING
|
||||
) CLUSTERED BY (id) INTO 100 buckets stored AS orc tblproperties ('transactional' = 'true');
|
||||
|
||||
-- Insert statement that takes into account the piwik_id of the openAIRE graph
|
||||
|
|
@ -32,7 +34,9 @@ SELECT substr(d1.id, 4) AS id,
|
|||
d1.latitude.value AS latitude,
|
||||
d1.longitude.value AS longitude,
|
||||
d1.websiteurl.value AS websiteurl,
|
||||
d1.openairecompatibility.classid AS compatibility
|
||||
d1.openairecompatibility.classid AS compatibility,
|
||||
d1.journal.issnprinted AS issn_printed,
|
||||
d1.journal.issnonline AS issn_online
|
||||
FROM ${openaire_db_name}.datasource d1
|
||||
LEFT OUTER JOIN
|
||||
(SELECT id, split(originalidd, '\\:')[1] as piwik_id
|
||||
|
|
@ -51,7 +55,7 @@ CREATE TABLE ${stats_db_name}.dual
|
|||
INSERT INTO ${stats_db_name}.dual
|
||||
VALUES ('X');
|
||||
INSERT INTO ${stats_db_name}.datasource_tmp (`id`, `name`, `type`, `dateofvalidation`, `yearofvalidation`, `harvested`,
|
||||
`piwik_id`, `latitude`, `longitude`, `websiteurl`, `compatibility`)
|
||||
`piwik_id`, `latitude`, `longitude`, `websiteurl`, `compatibility`, `issn_printed`, `issn_online`)
|
||||
SELECT 'other',
|
||||
'Other',
|
||||
'Repository',
|
||||
|
|
@ -62,7 +66,9 @@ SELECT 'other',
|
|||
NULL,
|
||||
NULL,
|
||||
NULL,
|
||||
'unknown'
|
||||
'unknown',
|
||||
null,
|
||||
null
|
||||
FROM ${stats_db_name}.dual
|
||||
WHERE 'other' not in (SELECT id FROM ${stats_db_name}.datasource_tmp WHERE name = 'Unknown Repository');
|
||||
DROP TABLE ${stats_db_name}.dual;
|
||||
|
|
@ -97,13 +103,4 @@ where d.datainfo.deletedbyinference = false;
|
|||
|
||||
CREATE OR REPLACE VIEW ${stats_db_name}.datasource_results AS
|
||||
SELECT datasource AS id, id AS result
|
||||
FROM ${stats_db_name}.result_datasources;
|
||||
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_tmp COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_tmp COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_languages COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_languages COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_oids COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_oids COMPUTE STATISTICS FOR COLUMNS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_organizations COMPUTE STATISTICS;
|
||||
-- ANALYZE TABLE ${stats_db_name}.datasource_organizations COMPUTE STATISTICS FOR COLUMNS;
|
||||
FROM ${stats_db_name}.result_datasources;
|
||||
Loading…
Reference in New Issue