forked from D-Net/dnet-hadoop
222 lines
9.8 KiB
SQL
222 lines
9.8 KiB
SQL
create table TARGET.indi_project_pubs_count stored as parquet as
|
|
select pr.id id, count(p.id) total_pubs from SOURCE.project_results pr
|
|
join SOURCE.publication p on p.id=pr.result
|
|
group by pr.id
|
|
|
|
create table TARGET.indi_project_datasets_count stored as parquet as
|
|
select pr.id id, count(d.id) total_datasets from SOURCE.project_results pr
|
|
join SOURCE.dataset d on d.id=pr.result
|
|
group by pr.id
|
|
|
|
create table TARGET.indi_project_software_count stored as parquet as
|
|
select pr.id id, count(s.id) total_software from SOURCE.project_results pr
|
|
join SOURCE.software s on s.id=pr.result
|
|
group by pr.id
|
|
|
|
create table TARGET.indi_project_otherresearch_count stored as parquet as
|
|
select pr.id id, count(o.id) total_other from SOURCE.project_results pr
|
|
join SOURCE.otherresearchproduct o on o.id=pr.result
|
|
group by pr.id
|
|
|
|
create table TARGET.indi_pub_avg_year_country_oa stored as parquet as
|
|
select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
|
|
round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
|
|
from
|
|
(SELECT year, country, SUM(CASE
|
|
WHEN bestlicence='Open Access' THEN 1
|
|
ELSE 0
|
|
END) AS OpenAccess, SUM(CASE
|
|
WHEN bestlicence<>'Open Access' THEN 1
|
|
ELSE 0
|
|
END) AS NonOpenAccess
|
|
FROM SOURCE.publication p
|
|
join SOURCE.result_organization ro on p.id=ro.id
|
|
join SOURCE.organization o on o.id=ro.organization
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by year, country) tmp
|
|
|
|
create table TARGET.indi_dataset_avg_year_country_oa stored as parquet as
|
|
select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
|
|
round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
|
|
from
|
|
(SELECT year, country, SUM(CASE
|
|
WHEN bestlicence='Open Access' THEN 1
|
|
ELSE 0
|
|
END) AS OpenAccess, SUM(CASE
|
|
WHEN bestlicence<>'Open Access' THEN 1
|
|
ELSE 0
|
|
END) AS NonOpenAccess
|
|
FROM SOURCE.dataset d
|
|
join SOURCE.result_organization ro on d.id=ro.id
|
|
join SOURCE.organization o on o.id=ro.organization
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by year, country) tmp
|
|
|
|
create table TARGET.indi_software_avg_year_country_oa stored as parquet as
|
|
select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
|
|
round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
|
|
from
|
|
(SELECT year, country, SUM(CASE
|
|
WHEN bestlicence='Open Access' THEN 1
|
|
ELSE 0
|
|
END) AS OpenAccess, SUM(CASE
|
|
WHEN bestlicence<>'Open Access' THEN 1
|
|
ELSE 0
|
|
END) AS NonOpenAccess
|
|
FROM SOURCE.software s
|
|
join SOURCE.result_organization ro on s.id=ro.id
|
|
join SOURCER.organization o on o.id=ro.organization
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by year, country) tmp
|
|
|
|
|
|
create table TARGET.indi_other_avg_year_country_oa stored as parquet as
|
|
select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
|
|
round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
|
|
from
|
|
(SELECT year, country, SUM(CASE
|
|
WHEN bestlicence='Open Access' THEN 1
|
|
ELSE 0
|
|
END) AS OpenAccess, SUM(CASE
|
|
WHEN bestlicence<>'Open Access' THEN 1
|
|
ELSE 0
|
|
END) AS NonOpenAccess
|
|
FROM SOURCE.otherresearchproduct orp
|
|
join SOURCE.result_organization ro on orp.id=ro.id
|
|
join SOURCE.organization o on o.id=ro.organization
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by year, country) tmp
|
|
|
|
create table TARGET.indi_pub_avg_year_context_oa stored as parquet as
|
|
with total as
|
|
(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from SOURCE.publication_concepts pc
|
|
join SOURCE.context c on pc.concept like concat('%',c.id,'%')
|
|
join SOURCE.publication p on p.id=pc.id
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by c.name, year )
|
|
select year, name, round(no_of_pubs/total*100,3) averageofpubs
|
|
from total
|
|
|
|
create table TARGET.indi_dataset_avg_year_context_oa stored as parquet as
|
|
with total as
|
|
(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from SOURCE.dataset_concepts pc
|
|
join SOURCE.context c on pc.concept like concat('%',c.id,'%')
|
|
join SOURCE.dataset p on p.id=pc.id
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by c.name, year )
|
|
select year, name, round(no_of_pubs/total*100,3) averageofdataset
|
|
from total
|
|
|
|
create table TARGET.indi_software_avg_year_context_oa stored as parquet as
|
|
with total as
|
|
(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from SOURCE.software_concepts pc
|
|
join SOURCE.context c on pc.concept like concat('%',c.id,'%')
|
|
join SOURCE.software p on p.id=pc.id
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by c.name, year )
|
|
select year, name, round(no_of_pubs/total*100,3) averageofsoftware
|
|
from total
|
|
|
|
create table TARGET.indi_other_avg_year_context_oa stored as parquet as
|
|
with total as
|
|
(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from SOURCE.otherresearchproduct_concepts pc
|
|
join SOURCE.context c on pc.concept like concat('%',c.id,'%')
|
|
join SOURCE.otherresearchproduct p on p.id=pc.id
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by c.name, year )
|
|
select year, name, round(no_of_pubs/total*100,3) averageofother
|
|
from total
|
|
|
|
create table TARGET.indi_other_avg_year_content_oa stored as parquet as
|
|
with total as
|
|
(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
|
|
from SOURCE.otherresearchproduct_datasources pd
|
|
join SOURCE.datasource d on datasource=d.id
|
|
join SOURCE.otherresearchproduct p on p.id=pd.id
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by d.type, year)
|
|
select year, type, round(no_of_pubs/total*100,3) averageOfOtherresearchproduct
|
|
from total
|
|
|
|
create table TARGET.indi_software_avg_year_content_oa stored as parquet as
|
|
with total as
|
|
(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
|
|
from SOURCE.software_datasources pd
|
|
join SOURCE.datasource d on datasource=d.id
|
|
join SOURCE.software p on p.id=pd.id
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by d.type, year)
|
|
select year, type, round(no_of_pubs/total*100,3) averageOfSoftware
|
|
from total
|
|
|
|
create table TARGET.indi_dataset_avg_year_content_oa stored as parquet as
|
|
with total as
|
|
(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
|
|
from SOURCE.dataset_datasources pd
|
|
join SOURCE.datasource d on datasource=d.id
|
|
join SOURCE.dataset p on p.id=pd.id
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by d.type, year)
|
|
select year, type, round(no_of_pubs/total*100,3) averageOfDatasets
|
|
from total
|
|
|
|
create table TARGET.indi_pub_avg_year_content_oa stored as parquet as
|
|
with total as
|
|
(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
|
|
from SOURCE.publication_datasources pd
|
|
join SOURCE.datasource d on datasource=d.id
|
|
join SOURCE.publication p on p.id=pd.id
|
|
where cast(year as int)>=2003 and cast(year as int)<=2021
|
|
group by d.type, year)
|
|
select year, type, round(no_of_pubs/total*100,3) averageOfPubs
|
|
from total
|
|
|
|
create table TARGET.indi_pub_has_cc_licence_tr stored as parquet as
|
|
select distinct p.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license_tr
|
|
from SOURCE.publication p
|
|
left outer join (select p.id, license.type as lic from SOURCE.publication p
|
|
join SOURCE.publication_licenses as license on license.id = p.id
|
|
where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp
|
|
on p.id= tmp.id
|
|
|
|
create table TARGET.indi_pub_has_cc_licence_url stored as parquet as
|
|
select distinct p.id, (case when lic_host='' or lic_host is null then 0 else 1 end) as has_cc_license_url
|
|
from SOURCE.publication p
|
|
left outer join (select p.id, lower(parse_url(license.type, "HOST")) as lic_host
|
|
from SOURCE.publication p
|
|
join SOURCE.publication_licenses as license on license.id = p.id
|
|
WHERE lower(parse_url(license.type, 'HOST')) = 'creativecommons.org') tmp
|
|
on p.id= tmp.id
|
|
|
|
create table TARGET.indi_pub_has_cc_licence_f stored as parquet as
|
|
select distinct p.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license_f
|
|
from SOURCE.publication p
|
|
left outer join (select p.id, license.type as lic from SOURCE.publication p
|
|
join SOURCE.publication_licenses as license on license.id = p.id
|
|
where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp
|
|
on p.id= tmp.id
|
|
|
|
create table TARGET.indi_pub_has_abstract stored as parquet as
|
|
select distinct publication.id, coalesce(abstract, 1) has_abstract
|
|
from SOURCE.publication
|
|
|
|
compute stats TARGET.indi_project_pubs_count;
|
|
compute stats TARGET.indi_project_datasets_count;
|
|
compute stats TARGET.indi_project_software_count;
|
|
compute stats TARGET.indi_project_otherresearch_count;
|
|
compute stats TARGET.indi_pub_avg_year_country_oa;
|
|
compute stats TARGET.indi_dataset_avg_year_country_oa;
|
|
compute stats TARGET.indi_software_avg_year_country_oa;
|
|
compute stats TARGET.indi_other_avg_year_country_oa;
|
|
compute stats TARGET.indi_pub_avg_year_context_oa;
|
|
compute stats TARGET.indi_dataset_avg_year_context_oa;
|
|
compute stats TARGET.indi_software_avg_year_context_oa;
|
|
compute stats TARGET.indi_other_avg_year_context_oa;
|
|
compute stats TARGET.indi_other_avg_year_content_oa;
|
|
compute stats TARGET.indi_software_avg_year_content_oa;
|
|
compute stats TARGET.indi_dataset_avg_year_content_oa;
|
|
compute stats TARGET.indi_pub_avg_year_content_oa;
|
|
compute stats TARGET.indi_pub_has_cc_licence_tr;
|
|
compute stats TARGET.indi_pub_has_cc_licence_url;
|
|
compute stats TARGET.indi_pub_has_cc_licence_f;
|
|
compute stats TARGET.indi_pub_has_abstract; |