dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql

@@ -27,6 +27,23 @@ select * from stats_ext.rndexpediture;
 create view ${stats_db_name}.issn_gold_oa_dataset as
 select * from stats_ext.issn_gold_oa_dataset;
 
+------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------------
+-- Usage statistics
+------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------------
+create or replace view ${stats_db_name}.usage_stats as
+select * from openaire_prod_usage_stats.usage_stats;
+
+create or replace view ${stats_db_name}.downloads_stats as
+select * from openaire_prod_usage_stats.downloads_stats;
+
+create or replace view ${stats_db_name}.pageviews_stats as
+select * from openaire_prod_usage_stats.pageviews_stats;
+
+create or replace view ${stats_db_name}.views_stats as
+select * from openaire_prod_usage_stats.views_stats;
+
 ------------------------------------------------------------------------------------------------
 ------------------------------------------------------------------------------------------------
 -- Creation date of the database
@@ -36,4 +53,4 @@ create table ${stats_db_name}.creation_date as
 select date_format(current_date(), 'dd-MM-yyyy') as date;
 --
 -- ANALYZE TABLE ${stats_db_name}.creation_date COMPUTE STATISTICS;
--- ANALYZE TABLE ${stats_db_name}.creation_date COMPUTE STATISTICS FOR COLUMNS;
+-- ANALYZE TABLE ${stats_db_name}.creation_date COMPUTE STATISTICS FOR COLUMNS;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql

@@ -14,7 +14,7 @@ LEFT OUTER JOIN
 (
     SELECT substr(d.id, 4) id 
     from ${openaire_db_name}.datasource d 
-    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_sources STORED AS PARQUET as
 SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
@@ -25,7 +25,7 @@ LEFT OUTER JOIN
 (
     SELECT substr(d.id, 4) id 
     from ${openaire_db_name}.datasource d 
-    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id;
     
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_sources STORED AS PARQUET as
 SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
@@ -36,7 +36,7 @@ LEFT OUTER JOIN
 (
     SELECT substr(d.id, 4) id 
     from ${openaire_db_name}.datasource d 
-    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id;
     
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_sources STORED AS PARQUET as
 SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
@@ -47,7 +47,7 @@ LEFT OUTER JOIN
 (
     SELECT substr(d.id, 4) id 
     from ${openaire_db_name}.datasource d 
-    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on p.datasource = d.id;
     
 CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_sources AS
 SELECT * FROM ${stats_db_name}.publication_sources
@@ -67,4 +67,17 @@ from (
     LATERAL VIEW explode(author) a as auth
     LATERAL VIEW explode(auth.pid) ap as auth_pid
     LATERAL VIEW explode(auth.pid.qualifier.classid) apt as author_pid_type
-    WHERE res.datainfo.deletedbyinference = FALSE and res.datainfo.invisible = FALSE and author_pid_type = 'orcid') as res
+    WHERE res.datainfo.deletedbyinference = FALSE and res.datainfo.invisible = FALSE and author_pid_type = 'orcid') as res;
+
+create table ${stats_db_name}.result_result stored as parquet as
+select substr(rel.source, 4) as source, substr(rel.target, 4) as target, relclass, subreltype
+from ${openaire_db_name}.relation rel
+join ${openaire_db_name}.result r1 on rel.source=r1.id
+join ${openaire_db_name}.result r2 on r2.id=rel.target
+where reltype='resultResult'
+    and r1.resulttype.classname!=r2.resulttype.classname
+    and r1.datainfo.deletedbyinference=false and r1.datainfo.invisible = FALSE
+    and r2.datainfo.deletedbyinference=false and r2.datainfo.invisible = FALSE
+    and r1.resulttype.classname != 'other'
+    and r2.resulttype.classname != 'other'
+    and rel.datainfo.deletedbyinference=false and rel.datainfo.invisible = FALSE;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql

@@ -8,22 +8,22 @@
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_licenses STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, licenses.value as type 
 from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.license) instances as licenses
-where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_licenses STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, licenses.value as type 
 from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.license) instances as licenses
-where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_licenses STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, licenses.value as type 
 from ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.license) instances as licenses
-where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_licenses STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, licenses.value as type 
 from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.license) instances as licenses
-where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false and p.datainfo.invisible = FALSE;
 
 CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_licenses AS
 SELECT * FROM ${stats_db_name}.publication_licenses
@@ -46,4 +46,17 @@ FROM (
     LEFT OUTER JOIN (
         SELECT substr(d.id, 4) id 
         from ${openaire_db_name}.datasource d 
-        WHERE d.datainfo.deletedbyinference=false) d on o.datasource = d.id;
+        WHERE d.datainfo.deletedbyinference=false and d.datainfo.invisible = FALSE) d on o.datasource = d.id;
+
+-- ANALYZE TABLE ${stats_db_name}.publication_licenses COMPUTE STATISTICS;
+-- ANALYZE TABLE ${stats_db_name}.publication_licenses COMPUTE STATISTICS FOR COLUMNS;
+-- ANALYZE TABLE ${stats_db_name}.dataset_licenses COMPUTE STATISTICS;
+-- ANALYZE TABLE ${stats_db_name}.dataset_licenses COMPUTE STATISTICS FOR COLUMNS;
+-- ANALYZE TABLE ${stats_db_name}.software_licenses COMPUTE STATISTICS;
+-- ANALYZE TABLE ${stats_db_name}.software_licenses COMPUTE STATISTICS FOR COLUMNS;
+-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_licenses COMPUTE STATISTICS;
+-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct_licenses COMPUTE STATISTICS FOR COLUMNS;
+-- ANALYZE TABLE ${stats_db_name}.organization_pids COMPUTE STATISTICS;
+-- ANALYZE TABLE ${stats_db_name}.organization_pids COMPUTE STATISTICS FOR COLUMNS;
+-- ANALYZE TABLE ${stats_db_name}.organization_sources COMPUTE STATISTICS;
+-- ANALYZE TABLE ${stats_db_name}.organization_sources COMPUTE STATISTICS FOR COLUMNS;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql

@@ -9,22 +9,22 @@
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_refereed STORED AS PARQUET as
 select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.publication r lateral view explode(r.instance) instances as inst
-where r.datainfo.deletedbyinference=false;
+where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_refereed STORED AS PARQUET as
 select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.dataset r lateral view explode(r.instance) instances as inst
-where r.datainfo.deletedbyinference=false;
+where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_refereed STORED AS PARQUET as
 select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.software r lateral view explode(r.instance) instances as inst
-where r.datainfo.deletedbyinference=false;
+where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE;
 
 CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_refereed STORED AS PARQUET as
 select substr(r.id, 4) as id, inst.refereed.classname as refereed
 from ${openaire_db_name}.otherresearchproduct r lateral view explode(r.instance) instances as inst
-where r.datainfo.deletedbyinference=false;
+where r.datainfo.deletedbyinference=false and r.datainfo.invisible = FALSE;
 
 CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_refereed as
 select * from ${stats_db_name}.publication_refereed

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16-createIndicatorsTables.sql

@@ -1,28 +1,29 @@
+---- Sprint 1 ----
 create table indi_pub_green_oa stored as parquet as
-select distinct p.id, coalesce(green_oa, 0) as green_oa 
+select distinct p.id, coalesce(green_oa, 0) as green_oa
 from publication p
-left outer join ( 
-select p.id, 1 as green_oa 
+left outer join (
+select p.id, 1 as green_oa
 from publication p
 join result_instance ri on ri.id = p.id
 join datasource on datasource.id = ri.hostedby
 where datasource.type like '%Repository%'
-and (ri.accessright = 'Open Access'  
-or ri.accessright = 'Embargo')) tmp 
+and (ri.accessright = 'Open Access'
+or ri.accessright = 'Embargo' or ri.accessright = 'Open Source')) tmp
 on p.id= tmp.id;
 
 create table indi_pub_grey_lit stored as parquet as
 select distinct p.id, coalesce(grey_lit, 0) as grey_lit
 from publication p
 left outer join (
-select p.id, 1 as grey_lit 
+select p.id, 1 as grey_lit
 from publication p
 join result_classifications rt on rt.id = p.id
-where rt.type not in ('Article','Part of book or chapter of book','Book','Doctoral thesis','Master thesis','Data Paper', 'Thesis', 'Bachelor thesis', 'Conference object') and 
+where rt.type not in ('Article','Part of book or chapter of book','Book','Doctoral thesis','Master thesis','Data Paper', 'Thesis', 'Bachelor thesis', 'Conference object') and
 not exists (select 1 from result_classifications rc where type ='Other literature type' and rc.id=p.id)) tmp on p.id=tmp.id;
 
 create table indi_pub_doi_from_crossref stored as parquet as
-select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref 
+select distinct p.id, coalesce(doi_from_crossref, 0) as doi_from_crossref
 from publication p
 left outer join
 (select ri.id, 1 as doi_from_crossref from result_instance ri
@@ -33,7 +34,7 @@ on tmp.id=p.id;
 create table indi_pub_gold_oa stored as parquet as
 select distinct p.id, coalesce(gold_oa, 0) as gold_oa
 from publication p
-left outer join ( 
+left outer join (
 select p.id, 1 as gold_oa
 from publication p
 join result_instance ri on ri.id = p.id
@@ -41,226 +42,56 @@ join datasource on datasource.id = ri.hostedby
 where datasource.id like '%doajarticles%') tmp
 on p.id= tmp.id;
 
-create table indi_project_pubs_count stored as parquet as
-select  pr.id id, count(p.id) total_pubs from project_results pr
-join publication p on p.id=pr.result
-group by pr.id;
-
-create table indi_project_datasets_count stored as parquet as
-select pr.id id, count(d.id) total_datasets from project_results pr
-join dataset d on d.id=pr.result
-group by pr.id;
-
-create table indi_project_software_count stored as parquet as
-select  pr.id id, count(s.id) total_software from project_results pr
-join software s on s.id=pr.result
-group by pr.id;
-
-create table indi_project_otherresearch_count stored as parquet as
-select pr.id id, count(o.id) total_other from project_results pr
-join otherresearchproduct o on o.id=pr.result
-group by pr.id;
-
-create table indi_pub_avg_year_country_oa stored as parquet as
-select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
-round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
- from
- (SELECT year, country, SUM(CASE
-    WHEN bestlicence='Open Access' THEN 1
- ELSE 0
- END) AS OpenAccess, SUM(CASE
- WHEN bestlicence<>'Open Access' THEN 1
- ELSE 0
- END) AS NonOpenAccess
- FROM publication p
- join result_organization ro on p.id=ro.id
- join organization o on o.id=ro.organization
- where cast(year as int)>=2003 and cast(year as int)<=2021
- group by year, country) tmp;
-
-create table indi_dataset_avg_year_country_oa stored as parquet as
-select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
-round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
- from
- (SELECT year, country, SUM(CASE
-    WHEN bestlicence='Open Access' THEN 1
- ELSE 0
- END) AS OpenAccess, SUM(CASE
- WHEN bestlicence<>'Open Access' THEN 1
- ELSE 0
- END) AS NonOpenAccess
- FROM dataset d
- join result_organization ro on d.id=ro.id
- join organization o on o.id=ro.organization
- where cast(year as int)>=2003 and cast(year as int)<=2021
- group by year, country) tmp;
-
-create table indi_software_avg_year_country_oa stored as parquet as
-select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
-round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
- from
- (SELECT year, country, SUM(CASE
-    WHEN bestlicence='Open Access' THEN 1
- ELSE 0
- END) AS OpenAccess, SUM(CASE
- WHEN bestlicence<>'Open Access' THEN 1
- ELSE 0
- END) AS NonOpenAccess
- FROM software s
- join result_organization ro on s.id=ro.id
- join organization o on o.id=ro.organization
- where cast(year as int)>=2003 and cast(year as int)<=2021
- group by year, country) tmp;
-
-
-create table indi_other_avg_year_country_oa stored as parquet as
-select year, country, round(OpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageOA,
-round(NonOpenAccess/(OpenAccess+NonOpenAccess)*100,3) as averageNonOA
- from
- (SELECT year, country, SUM(CASE
-    WHEN bestlicence='Open Access' THEN 1
- ELSE 0
- END) AS OpenAccess, SUM(CASE
- WHEN bestlicence<>'Open Access' THEN 1
- ELSE 0
- END) AS NonOpenAccess
- FROM otherresearchproduct orp
- join result_organization ro on orp.id=ro.id
- join organization o on o.id=ro.organization
- where cast(year as int)>=2003 and cast(year as int)<=2021
- group by year, country) tmp;
-
-create table indi_pub_avg_year_context_oa stored as parquet as
-with total as
-(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from publication_concepts pc
-join context c on pc.concept like concat('%',c.id,'%')
-join publication p on p.id=pc.id
-where cast(year as int)>=2003 and cast(year as int)<=2021
-group by c.name, year )
-select year, name, round(no_of_pubs/total*100,3) averageofpubs
-from total;
-
-create table indi_dataset_avg_year_context_oa stored as parquet as
-with total as
-(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from dataset_concepts pc
-join context c on pc.concept like concat('%',c.id,'%')
-join dataset p on p.id=pc.id
-where cast(year as int)>=2003 and cast(year as int)<=2021
-group by c.name, year )
-select year, name, round(no_of_pubs/total*100,3) averageofdataset
-from total;
-
-create table indi_software_avg_year_context_oa stored as parquet as
-with total as
-(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from software_concepts pc
-join context c on pc.concept like concat('%',c.id,'%')
-join software p on p.id=pc.id
-where cast(year as int)>=2003 and cast(year as int)<=2021
-group by c.name, year )
-select year, name, round(no_of_pubs/total*100,3) averageofsoftware
-from total;
-
-create table indi_other_avg_year_context_oa stored as parquet as
-with total as
-(select count(distinct pc.id) no_of_pubs, year, c.name name, sum(count(distinct pc.id)) over(PARTITION by year) as total from otherresearchproduct_concepts pc
-join context c on pc.concept like concat('%',c.id,'%')
-join otherresearchproduct p on p.id=pc.id
-where cast(year as int)>=2003 and cast(year as int)<=2021
-group by c.name, year )
-select year, name, round(no_of_pubs/total*100,3) averageofother
-from total;
-
-create table indi_other_avg_year_content_oa stored as parquet as
-with total as
-(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
-from otherresearchproduct_datasources pd
-join datasource d on datasource=d.id
-join otherresearchproduct p on p.id=pd.id
-where cast(year as int)>=2003 and cast(year as int)<=2021
-group by d.type, year)
-select year, type, round(no_of_pubs/total*100,3) averageOfOtherresearchproduct
-from total;
-
-create table indi_software_avg_year_content_oa stored as parquet as
-with total as
-(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
-from software_datasources pd
-join datasource d on datasource=d.id
-join software p on p.id=pd.id
-where cast(year as int)>=2003 and cast(year as int)<=2021
-group by d.type, year)
-select year, type, round(no_of_pubs/total*100,3) averageOfSoftware
-from total;
-
-create table indi_dataset_avg_year_content_oa stored as parquet as
-with total as
-(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
-from dataset_datasources pd
-join datasource d on datasource=d.id
-join dataset p on p.id=pd.id
-where cast(year as int)>=2003 and cast(year as int)<=2021
-group by d.type, year)
-select year, type, round(no_of_pubs/total*100,3) averageOfDatasets
-from total;
-
-create table indi_pub_avg_year_content_oa stored as parquet as
-with total as
-(select count(distinct pd.id) no_of_pubs, year, d.type type, sum(count(distinct pd.id)) over(PARTITION by year) as total
-from publication_datasources pd
-join datasource d on datasource=d.id
-join publication p on p.id=pd.id
-where cast(year as int)>=2003 and cast(year as int)<=2021
-group by d.type, year)
-select year, type, round(no_of_pubs/total*100,3) averageOfPubs
-from total;
-
-create table indi_pub_has_cc_licence stored as parquet as
-select distinct p.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license
-from publication p
-left outer join (select p.id, license.type as lic from publication p
-join publication_licenses as license on license.id = p.id
+---- Sprint 2 ----
+create table indi_result_has_cc_licence stored as parquet as
+select distinct r.id, (case when lic='' or lic is null then 0 else 1 end) as has_cc_license
+from result r
+left outer join (select r.id, license.type as lic from result r
+join result_licenses as license on license.id = r.id
 where lower(license.type) LIKE '%creativecommons.org%' OR lower(license.type) LIKE '%cc-%') tmp
-on p.id= tmp.id;
+on r.id= tmp.id;
 
-create table indi_pub_has_cc_licence_url stored as parquet as
-select distinct p.id, (case when lic_host='' or lic_host is null then 0 else 1 end) as has_cc_license_url
-from publication p
-left outer join (select p.id, lower(parse_url(license.type, "HOST")) as lic_host
-from publication p
-join publication_licenses as license on license.id = p.id
-WHERE lower(parse_url(license.type, 'HOST')) = 'creativecommons.org') tmp
-on p.id= tmp.id;
+create table indi_result_has_cc_licence_url stored as parquet as
+select distinct r.id, case when lic_host='' or lic_host is null then 0 else 1 end as has_cc_license_url
+from result r
+left outer join (select r.id, lower(parse_url(license.type, "HOST")) as lic_host
+from result r
+join result_licenses as license on license.id = r.id
+WHERE lower(parse_url(license.type, "HOST")) = "creativecommons.org") tmp
+on r.id= tmp.id;
 
 create table indi_pub_has_abstract stored as parquet as
 select distinct publication.id, coalesce(abstract, 1) has_abstract
 from publication;
 
-create table indi_with_orcid stored as parquet as  
+create table indi_result_with_orcid stored as parquet as
 select distinct r.id, coalesce(has_orcid, 0) as has_orcid
-from result r 
-left outer join (select id, 1 as has_orcid from result_orcid) tmp 
-on r.id= tmp.id 
+from result r
+left outer join (select id, 1 as has_orcid from result_orcid) tmp
+on r.id= tmp.id;
 
-create table indi_funded_result_with_fundref stored as parquet as  
+
+---- Sprint 3 ----
+create table indi_funded_result_with_fundref stored as parquet as
 select distinct r.id, coalesce(fundref, 0) as fundref
-from project_results r 
+from project_results r
 left outer join (select distinct id, 1 as fundref from project_results
-where provenance='Harvested') tmp 
-on r.id= tmp.id
+where provenance='Harvested') tmp
+on r.id= tmp.id;
 
-create table indi_result_org_country_collab stored as parquet as  
-with tmp as 
+create table indi_result_org_country_collab stored as parquet as
+with tmp as
 (select o.id as id, o.country , ro.id as result,r.type  from organization o
 join result_organization ro on o.id=ro.organization
 join result r on r.id=ro.id where o.country <> 'UNKNOWN')
 select o1.id org1,o2.country country2, o1.type, count(distinct o1.result) as collaborations
 from tmp as o1
 join tmp as o2 on o1.result=o2.result
-where o1.id<>o2.id and o1.country<>o2.country 
-group by o1.id, o1.type,o2.country
+where o1.id<>o2.id and o1.country<>o2.country
+group by o1.id, o1.type,o2.country;
 
-create table indi_result_org_collab stored as parquet as  
-with tmp as 
+create table indi_result_org_collab stored as parquet as
+with tmp as
 (select o.id, ro.id as result,r.type  from organization o
 join result_organization ro on o.id=ro.organization
 join result r on r.id=ro.id)
@@ -268,15 +99,82 @@ select o1.id org1,o2.id org2, o1.type, count(distinct o1.result) as collaboratio
 from tmp as o1
 join tmp as o2 on o1.result=o2.result
 where o1.id<>o2.id
-group by o1.id, o2.id, o1.type
+group by o1.id, o2.id, o1.type;
 
-create table indi_result_org_country_collab stored as parquet as  
-with tmp as 
-(select o.id as id, o.country , ro.id as result,r.type  from organization o
+create table indi_funder_country_collab stored as parquet as
+with tmp as (select funder, project, country from organization_projects op
+join organization o on o.id=op.id
+join project p on p.id=op.project
+where country <> 'UNKNOWN')
+select f1.funder, f1.country as country1, f2.country as country2, count(distinct f1.project) as collaborations
+from tmp as f1
+join tmp as f2 on f1.project=f2.project
+where f1.country<>f2.country
+group by f1.funder, f2.country, f1.country;
+
+create table indi_result_country_collab stored as parquet as
+with tmp as
+(select country, ro.id as result,r.type  from organization o
 join result_organization ro on o.id=ro.organization
-join result r on r.id=ro.id where o.country <> 'UNKNOWN')
-select o1.id org1,o2.country country2, o1.type, count(distinct o1.result) as collaborations
+join result r on r.id=ro.id)
+select o1.country country1, o2.country country2, o1.type, count(distinct o1.result) as collaborations
 from tmp as o1
 join tmp as o2 on o1.result=o2.result
-where o1.id<>o2.id and o1.country<>o2.country 
-group by o1.id, o1.type,o2.country
+where o1.country<>o2.country
+group by o1.country, o2.country, o1.type;
+
+---- Sprint 4 ----
+create table indi_pub_diamond stored as parquet as
+select distinct pd.id, coalesce(in_diamond_journal, 0) as in_diamond_journal
+from publication_datasources pd
+left outer join (
+select pd.id, 1 as in_diamond_journal from publication_datasources pd
+join datasource d on d.id=pd.datasource
+join stats_ext.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online)
+and (ps.journal_is_in_doaj=true or ps.journal_is_oa=true) and ps.has_apc=false) tmp
+on pd.id=tmp.id;
+
+create table indi_pub_hybrid stored as parquet as
+select distinct pd.id, coalesce(is_hybrid, 0) as is_hybrid
+from publication_datasources pd
+left outer join (
+select pd.id, 1 as is_hybrid from publication_datasources pd
+join datasource d on d.id=pd.datasource
+join stats_ext.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online)
+and (ps.journal_is_in_doaj=false and ps.journal_is_oa=false)) tmp
+on pd.id=tmp.id;
+
+create table indi_is_gold_oa stored as parquet as
+(select distinct pd.id, coalesce(gold_oa, 0) as gold_oa
+from publication_datasources pd
+left outer join (
+select pd.id, 1 as gold_oa from publication_datasources pd
+join datasource d on d.id=pd.datasource
+join stats_ext.plan_s_jn ps on (ps.issn_print=d.issn_printed or ps.issn_online=d.issn_online)
+where ps.journal_is_in_doaj is true or ps.journal_is_oa is true) tmp
+on pd.id=tmp.id);
+
+create table indi_pub_in_transformative stored as parquet as
+select distinct pd.id, coalesce(is_transformative, 0) as is_transformative
+from publication pd
+left outer join (
+select  pd.id, 1 as is_transformative from publication_datasources pd
+join datasource d on d.id=pd.datasource
+join stats_ext.plan_s_jn ps where (ps.issn_print=d.issn_printed and ps.issn_online=d.issn_online)
+and ps.is_transformative_journal=true) tmp
+on pd.id=tmp.id;
+
+create table indi_pub_closed_other_open stored as parquet as
+select distinct ri.id, coalesce(pub_closed_other_open, 0) as pub_closed_other_open from result_instance ri
+left outer join
+(select ri.id, 1 as pub_closed_other_open from result_instance ri
+join publication p on p.id=ri.id
+join datasource d on ri.hostedby=d.id
+where d.type like '%Journal%' and ri.accessright='Closed Access' and
+(p.bestlicence='Open Access' or p.bestlicence='Open Source')) tmp
+on tmp.id=ri.id;
+
+
+---- Sprint 5 ----
+create table indi_result_no_of_copies stored as parquet as
+select id, count(id) as number_of_copies from result_instance group by id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2.sql

@@ -38,13 +38,13 @@ SELECT substr(p.id, 4)                                            as id,
        case when size(p.description) > 0 then true else false end as abstract,
        'publication'                                              as type
 from ${openaire_db_name}.publication p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_classifications STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, instancetype.classname as type
 from ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.instance.instancetype) instances as instancetype
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_concepts STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, case
@@ -53,45 +53,45 @@ SELECT substr(p.id, 4) as id, case
     when contexts.context.id RLIKE '^[^::]+$' then concat(contexts.context.id, '::other::other') END as concept
 from ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.context) contexts as context
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_datasources STORED AS PARQUET as
 SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource
 FROM (
          SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource
          from ${openaire_db_name}.publication p lateral view explode(p.instance) instances as instance
-         where p.datainfo.deletedbyinference = false) p
+         where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false) p
          LEFT OUTER JOIN (
     SELECT substr(d.id, 4) id
     from ${openaire_db_name}.datasource d
-    WHERE d.datainfo.deletedbyinference = false) d on p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d on p.datasource = d.id;
 
 CREATE TABLE ${stats_db_name}.publication_languages STORED AS PARQUET AS
 select substr(p.id, 4) as id, p.language.classname as language
 FROM ${openaire_db_name}.publication p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_oids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
 FROM ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.originalid) oids AS ids
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_pids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value as pid
 FROM ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.pid) pids AS ppid
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_topics STORED AS PARQUET as
 select substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS TYPE, subjects.subject.value AS topic
 FROM ${openaire_db_name}.publication p
          LATERAL VIEW explode(p.subject) subjects AS subject
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.publication_citations STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
 FROM ${openaire_db_name}.publication p
          lateral view explode(p.extrainfo) citations AS citation
 WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
-  and p.datainfo.deletedbyinference = false;
+  and p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step20-createMonitorDB.sql

@@ -49,8 +49,10 @@ compute stats TARGET.result_greenoa;
 create table TARGET.result_languages stored as parquet as select * from SOURCE.result_languages orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.result_languages;
 
-create table TARGET.result_licences stored as parquet as select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result r where r.id=orig.id);
-compute stats TARGET.result_licences;
+create table TARGET.result_licenses stored as parquet as select * from SOURCE.result_licenses orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.result_licenses;
+
+create table TARGET.licenses_normalized as select * from SOURCE.licenses_normalized;
 
 create table TARGET.result_oids stored as parquet as select * from SOURCE.result_oids orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.result_oids;
@@ -79,6 +81,13 @@ compute stats TARGET.result_sources;
 create table TARGET.result_topics stored as parquet as select * from SOURCE.result_topics orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.result_topics;
 
+create view TARGET.foo1 as select * from SOURCE.result_result rr where rr.source in (select id from TARGET.result);
+create view TARGET.foo2 as select * from SOURCE.result_result rr where rr.target in (select id from TARGET.result);
+create table TARGET.result_result as select distinct * from (select * from TARGET.foo1 union all select * from TARGET.foo2) foufou;
+drop view TARGET.foo1;
+drop view TARGET.foo2;
+compute stats TARGET.result_result;
+
 -- datasources
 create view if not exists TARGET.datasource as select * from SOURCE.datasource;
 create view if not exists TARGET.datasource_oids as select * from SOURCE.datasource_oids;
@@ -100,28 +109,12 @@ create view if not exists TARGET.project as select * from SOURCE.project;
 create view if not exists TARGET.project_oids as select * from SOURCE.project_oids;
 create view if not exists TARGET.project_organizations as select * from SOURCE.project_organizations;
 create view if not exists TARGET.project_resultcount as select * from SOURCE.project_resultcount;
+create view if not exists TARGET.project_classification as select * from SOURCE.project_classification;
 
 create table TARGET.project_results stored as parquet as select id as result, project as id from TARGET.result_projects;
 compute stats TARGET.project_results;
 
 -- indicators
-create view TARGET.indi_dataset_avg_year_content_oa as select * from SOURCE.indi_dataset_avg_year_content_oa orig;
-create view TARGET.indi_dataset_avg_year_context_oa as select * from SOURCE.indi_dataset_avg_year_context_oa orig;
-create view TARGET.indi_dataset_avg_year_country_oa as select * from SOURCE.indi_dataset_avg_year_country_oa orig;
-
-create view TARGET.indi_other_avg_year_content_oa as select * from SOURCE.indi_other_avg_year_content_oa orig;
-create view TARGET.indi_other_avg_year_context_oa as select * from SOURCE.indi_other_avg_year_context_oa orig;
-create view TARGET.indi_other_avg_year_country_oa as select * from SOURCE.indi_other_avg_year_country_oa orig;
-
-create view TARGET.indi_project_datasets_count as select * from SOURCE.indi_project_datasets_count orig;
-create view TARGET.indi_project_otherresearch_count as select * from SOURCE.indi_project_otherresearch_count orig;
-create view TARGET.indi_project_pubs_count as select * from SOURCE.indi_project_pubs_count orig;
-create view TARGET.indi_project_software_count as select * from SOURCE.indi_project_software_count orig;
-
-create view TARGET.indi_pub_avg_year_content_oa as select * from SOURCE.indi_pub_avg_year_content_oa orig;
-create view TARGET.indi_pub_avg_year_context_oa as select * from SOURCE.indi_pub_avg_year_context_oa orig;
-create view TARGET.indi_pub_avg_year_country_oa as select * from SOURCE.indi_pub_avg_year_country_oa orig;
-
 create table TARGET.indi_pub_green_oa stored as parquet as select * from SOURCE.indi_pub_green_oa orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.indi_pub_green_oa;
 create table TARGET.indi_pub_grey_lit stored as parquet as select * from SOURCE.indi_pub_grey_lit orig where exists (select 1 from TARGET.result r where r.id=orig.id);
@@ -132,14 +125,31 @@ create table TARGET.indi_pub_gold_oa stored as parquet as select * from SOURCE.i
 compute stats TARGET.indi_pub_gold_oa;
 create table TARGET.indi_pub_has_abstract stored as parquet as select * from SOURCE.indi_pub_has_abstract orig where exists (select 1 from TARGET.result r where r.id=orig.id);
 compute stats TARGET.indi_pub_has_abstract;
-create table TARGET.indi_pub_has_cc_licence stored as parquet as select * from SOURCE.indi_pub_has_cc_licence orig where exists (select 1 from TARGET.result r where r.id=orig.id);
-compute stats TARGET.indi_pub_has_cc_licence;
-create table TARGET.indi_pub_has_cc_licence_url stored as parquet as select * from SOURCE.indi_pub_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id);
-compute stats TARGET.indi_pub_has_cc_licence_url;
+create table TARGET.indi_result_has_cc_licence stored as parquet as select * from SOURCE.indi_result_has_cc_licence orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_result_has_cc_licence;
+create table TARGET.indi_result_has_cc_licence_url stored as parquet as select * from SOURCE.indi_result_has_cc_licence_url orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_result_has_cc_licence_url;
 
-create view TARGET.indi_software_avg_year_content_oa as select * from SOURCE.indi_software_avg_year_content_oa orig;
-create view TARGET.indi_software_avg_year_context_oa as select * from SOURCE.indi_software_avg_year_context_oa orig;
-create view TARGET.indi_software_avg_year_country_oa as select * from SOURCE.indi_software_avg_year_country_oa orig;
+create view TARGET.indi_funder_country_collab as select * from SOURCE.indi_funder_country_collab;
+
+create table TARGET.indi_result_with_orcid stored as parquet as select * from SOURCE.indi_result_with_orcid orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_result_with_orcid;
+create table TARGET.indi_funded_result_with_fundref stored as parquet as select * from SOURCE.indi_funded_result_with_fundref orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_funded_result_with_fundref;
+create table TARGET.indi_pub_diamond stored as parquet as select * from SOURCE.indi_pub_diamond orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_pub_diamond;
+create table TARGET.indi_pub_hybrid stored as parquet as select * from SOURCE.indi_pub_hybrid orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_pub_hybrid;
+create table TARGET.indi_pub_in_transformative stored as parquet as select * from SOURCE.indi_pub_in_transformative orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_pub_in_transformative;
+create table TARGET.indi_pub_closed_other_open stored as parquet as select * from SOURCE.indi_pub_closed_other_open orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_pub_closed_other_open;
+
+create table TARGET.indi_result_no_of_copies stored as parquet as select * from SOURCE.indi_result_no_of_copies orig where exists (select 1 from TARGET.result r where r.id=orig.id);
+compute stats TARGET.indi_result_no_of_copies;
+
+--- Usage statistics
+create table TARGET.usage_stats stored as parquet as select * from SOURCE.usage_stats orig where exists (select 1 from TARGET.result r where r.id=orig.result_id);
 
 --denorm
 alter table TARGET.result rename to TARGET.res_tmp;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql

@@ -38,20 +38,20 @@ SELECT substr(d.id, 4)                                            AS id,
        CASE WHEN SIZE(d.description) > 0 THEN TRUE ELSE FALSE end AS abstract,
        'dataset'                                                  AS type
 FROM ${openaire_db_name}.dataset d
-WHERE d.datainfo.deletedbyinference = FALSE;
+WHERE d.datainfo.deletedbyinference = FALSE and d.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_citations STORED AS PARQUET AS
 SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
 FROM ${openaire_db_name}.dataset d
          LATERAL VIEW explode(d.extrainfo) citations AS citation
 WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
-  and d.datainfo.deletedbyinference = false;
+  and d.datainfo.deletedbyinference = false and d.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_classifications STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, instancetype.classname AS type
 FROM ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_concepts STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, case
@@ -60,7 +60,7 @@ SELECT substr(p.id, 4) as id, case
                                   when contexts.context.id RLIKE '^[^::]+$' then concat(contexts.context.id, '::other::other') END as concept
 from ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.context) contexts as context
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_datasources STORED AS PARQUET AS
 SELECT p.id, case when d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource
@@ -68,31 +68,31 @@ FROM (
          SELECT substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) AS datasource
          FROM ${openaire_db_name}.dataset p
                   LATERAL VIEW explode(p.instance) instances AS instance
-         where p.datainfo.deletedbyinference = false) p
+         where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false) p
          LEFT OUTER JOIN (
     SELECT substr(d.id, 4) id
     FROM ${openaire_db_name}.datasource d
-    WHERE d.datainfo.deletedbyinference = false) d ON p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d ON p.datasource = d.id;
 
 CREATE TABLE ${stats_db_name}.dataset_languages STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, p.language.classname AS language
 FROM ${openaire_db_name}.dataset p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_oids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
 FROM ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.originalid) oids AS ids
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_pids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid
 FROM ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.pid) pids AS ppid
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.dataset_topics STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
 FROM ${openaire_db_name}.dataset p
          LATERAL VIEW explode(p.subject) subjects AS subject
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql

@@ -38,20 +38,20 @@ SELECT substr(s.id, 4)                                            as id,
        CASE WHEN SIZE(s.description) > 0 THEN TRUE ELSE FALSE END AS abstract,
        'software'                                                 as type
 from ${openaire_db_name}.software s
-where s.datainfo.deletedbyinference = false;
+where s.datainfo.deletedbyinference = false and s.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_citations STORED AS PARQUET AS
 SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
 FROM ${openaire_db_name}.software s
          LATERAL VIEW explode(s.extrainfo) citations as citation
 where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
-  and s.datainfo.deletedbyinference = false;
+  and s.datainfo.deletedbyinference = false and s.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_classifications STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, instancetype.classname AS type
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_concepts STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, case
@@ -60,7 +60,7 @@ SELECT substr(p.id, 4) as id, case
                                   when contexts.context.id RLIKE '^[^::]+$' then concat(contexts.context.id, '::other::other') END as concept
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.context) contexts AS context
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_datasources STORED AS PARQUET AS
 SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource end as datasource
@@ -68,31 +68,31 @@ FROM (
          SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource
          FROM ${openaire_db_name}.software p
                   LATERAL VIEW explode(p.instance) instances AS instance
-         where p.datainfo.deletedbyinference = false) p
+         where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false) p
          LEFT OUTER JOIN (
     SELECT substr(d.id, 4) id
     FROM ${openaire_db_name}.datasource d
-    WHERE d.datainfo.deletedbyinference = false) d ON p.datasource = d.id;
+    WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d ON p.datasource = d.id;
 
 CREATE TABLE ${stats_db_name}.software_languages STORED AS PARQUET AS
 select substr(p.id, 4) AS id, p.language.classname AS language
 FROM ${openaire_db_name}.software p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_oids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.originalid) oids AS ids
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_pids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.pid) pids AS ppid
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.software_topics STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
 FROM ${openaire_db_name}.software p
          LATERAL VIEW explode(p.subject) subjects AS subject
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql

@@ -37,19 +37,19 @@ SELECT substr(o.id, 4)                                            AS id,
        CASE WHEN SIZE(o.description) > 0 THEN TRUE ELSE FALSE END AS abstract,
        'other'                                                    AS type
 FROM ${openaire_db_name}.otherresearchproduct o
-WHERE o.datainfo.deletedbyinference = FALSE;
+WHERE o.datainfo.deletedbyinference = FALSE and o.datainfo.invisible=false;
 
 -- Otherresearchproduct_citations
 CREATE TABLE ${stats_db_name}.otherresearchproduct_citations STORED AS PARQUET AS
 SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS cites
 FROM ${openaire_db_name}.otherresearchproduct o LATERAL VIEW explode(o.extrainfo) citations AS citation
 WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") != ""
-  and o.datainfo.deletedbyinference = false;
+  and o.datainfo.deletedbyinference = false and o.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_classifications STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, instancetype.classname AS type
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts STORED AS PARQUET AS
 SELECT substr(p.id, 4) as id, case
@@ -57,33 +57,33 @@ SELECT substr(p.id, 4) as id, case
                                   when contexts.context.id RLIKE '^[^::]+::[^::]+$' then concat(contexts.context.id, '::other')
                                   when contexts.context.id RLIKE '^[^::]+$' then concat(contexts.context.id, '::other::other') END as concept
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts AS context
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_datasources STORED AS PARQUET AS
 SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource
 FROM (SELECT substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource
       from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.instance) instances as instance
-      where p.datainfo.deletedbyinference = false) p
+      where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false) p
          LEFT OUTER JOIN(SELECT substr(d.id, 4) id
                          from ${openaire_db_name}.datasource d
-                         WHERE d.datainfo.deletedbyinference = false) d on p.datasource = d.id;
+                         WHERE d.datainfo.deletedbyinference = false and d.datainfo.invisible=false) d on p.datasource = d.id;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_languages STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, p.language.classname AS language
 FROM ${openaire_db_name}.otherresearchproduct p
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_oids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids AS ids
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_pids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.pid) pids AS ppid
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.otherresearchproduct_topics STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic
 FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject
-where p.datainfo.deletedbyinference = false;
+where p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql

@@ -5,24 +5,26 @@
 ------------------------------------------------------
 CREATE TABLE ${stats_db_name}.project_oids STORED AS PARQUET AS
 SELECT substr(p.id, 4) AS id, oids.ids AS oid
-FROM ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids AS ids;
+FROM ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids AS ids
+where p.datainfo.deletedbyinference=false  and p.datainfo.invisible=false;
+
 CREATE TABLE ${stats_db_name}.project_organizations AS
 SELECT substr(r.source, 4) AS id, substr(r.target, 4) AS organization
 from ${openaire_db_name}.relation r
 WHERE r.reltype = 'projectOrganization'
-  and r.datainfo.deletedbyinference = false;
+  and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.project_results STORED AS PARQUET AS
 SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS result, r.datainfo.provenanceaction.classname as provenance
 FROM ${openaire_db_name}.relation r
 WHERE r.reltype = 'resultProject'
-  and r.datainfo.deletedbyinference = false;
+  and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false;
 
 create table ${stats_db_name}.project_classification STORED AS PARQUET as
 select substr(p.id, 4) as id, class.h2020programme.code, class.level1, class.level2, class.level3
 from ${openaire_db_name}.project p
     lateral view explode(p.h2020classification) classifs as class
-where p.datainfo.deletedbyinference=false and class.h2020programme is not null;
+where p.datainfo.deletedbyinference=false and p.datainfo.invisible=false and class.h2020programme is not null;
 
 CREATE TABLE ${stats_db_name}.project_tmp
 (
@@ -72,7 +74,7 @@ SELECT substr(p.id, 4)                                                 AS id,
        p.code.value                                                    AS code,
        p.totalcost                                                     AS totalcost
 FROM ${openaire_db_name}.project p
-WHERE p.datainfo.deletedbyinference = false;
+WHERE p.datainfo.deletedbyinference = false and p.datainfo.invisible=false;
 
 create table ${stats_db_name}.funder STORED AS PARQUET as
 select distinct xpath_string(fund, '//funder/id')        as id,

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql

@@ -127,7 +127,7 @@ CREATE TABLE ${stats_db_name}.result_organization STORED AS PARQUET AS
 SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization
 FROM ${openaire_db_name}.relation r
 WHERE r.reltype = 'resultOrganization'
-  and r.datainfo.deletedbyinference = false;
+  and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.result_projects STORED AS PARQUET AS
 select pr.result AS id, pr.id AS project, datediff(p.enddate, p.startdate) AS daysfromend, pr.provenance as provenance

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql

@@ -44,7 +44,7 @@ FROM ${openaire_db_name}.datasource d1
                LATERAL VIEW EXPLODE(originalid) temp AS originalidd
       WHERE originalidd like "piwik:%") AS d2
      ON d1.id = d2.id
-WHERE d1.datainfo.deletedbyinference = FALSE;
+WHERE d1.datainfo.deletedbyinference = FALSE and d1.datainfo.invisible=false;
 
 -- Updating temporary table with everything that is not based on results -> This is done with the following "dual" table.
 -- Creating a temporary dual table that will be removed after the following insert
@@ -76,24 +76,26 @@ UPDATE ${stats_db_name}.datasource_tmp SET yearofvalidation=null WHERE yearofval
 
 CREATE TABLE ${stats_db_name}.datasource_languages STORED AS PARQUET AS
 SELECT substr(d.id, 4) AS id, langs.languages AS language
-FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.odlanguages.value) langs AS languages;
+FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.odlanguages.value) langs AS languages
+where d.datainfo.deletedbyinference=false and d.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.datasource_oids STORED AS PARQUET AS
 SELECT substr(d.id, 4) AS id, oids.ids AS oid
-FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.originalid) oids AS ids;
+FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.originalid) oids AS ids
+where d.datainfo.deletedbyinference=false and d.datainfo.invisible=false;
 
 CREATE TABLE ${stats_db_name}.datasource_organizations STORED AS PARQUET AS
 SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization
 FROM ${openaire_db_name}.relation r
-WHERE r.reltype = 'datasourceOrganization' and r.datainfo.deletedbyinference = false;
+WHERE r.reltype = 'datasourceOrganization' and r.datainfo.deletedbyinference = false and r.datainfo.invisible=false;
 
 -- datasource sources:
 -- where the datasource info have been collected from.
 create table if not exists ${stats_db_name}.datasource_sources STORED AS PARQUET AS
 select substr(d.id, 4) as id, substr(cf.key, 4) as datasource
 from ${openaire_db_name}.datasource d lateral view explode(d.collectedfrom) cfrom as cf
-where d.datainfo.deletedbyinference = false;
+where d.datainfo.deletedbyinference = false and d.datainfo.invisible=false;
 
 CREATE OR REPLACE VIEW ${stats_db_name}.datasource_results AS
 SELECT datasource AS id, id AS result
-FROM ${stats_db_name}.result_datasources;
+FROM ${stats_db_name}.result_datasources;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql

@@ -9,7 +9,7 @@ SELECT substr(o.id, 4)        as id,
        o.legalshortname.value as legalshortname,
        o.country.classid      as country
 FROM ${openaire_db_name}.organization o
-WHERE o.datainfo.deletedbyinference = FALSE;
+WHERE o.datainfo.deletedbyinference = FALSE and o.datainfo.invisible = FALSE;
 
 CREATE OR REPLACE VIEW ${stats_db_name}.organization_datasources AS
 SELECT organization AS id, id AS datasource