Merge pull request 'stats_wf_extensions_and_corrections' (#28) from spyros/dnet-hadoop:stats_wf_extensions_and_corrections into master

Thank you Guys! The update workflow will be made available to the beta & production orchestration systems under the HDFS path

```/lib/dnet/oa/graph/stats/oozie_app```

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/config-default.xml

@@ -16,11 +16,11 @@
         <value>spark2</value>
     </property>
     <property>
-        <name>hiveMetastoreUris</name>
+        <name>hive_metastore_uris</name>
         <value>thrift://iis-cdh5-test-m3.ocean.icm.edu.pl:9083</value>
     </property>
     <property>
-        <name>hiveJdbcUrl</name>
+        <name>hive_jdbc_url</name>
         <value>jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000</value>
     </property>
 	<property>

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/impala-shell.sh

@@ -0,0 +1,18 @@
+export PYTHON_EGG_CACHE=/home/$(whoami)/.python-eggs
+export link_folder=/tmp/impala-shell-python-egg-cache-$(whoami)
+if ! [ -L $link_folder ]
+then
+    rm -Rf "$link_folder"
+    ln -sfn ${PYTHON_EGG_CACHE}${link_folder} ${link_folder}
+fi
+
+echo "Getting file from " $3
+hdfs dfs -copyToLocal $3
+
+echo "Running impala shell make the new database visible"
+impala-shell -q "INVALIDATE METADATA;"
+
+echo "Running impala shell to compute new table stats"
+impala-shell -d $1 -f $2
+echo "Impala shell finished"
+rm $2

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step1.sql

@@ -1,11 +1,8 @@
--- DROP database if EXISTS ${hive_db_name} cascade;
+--------------------------------------------------------------
--- CREATE database ${hive_db_name};
+--------------------------------------------------------------
--- 
+-- Stats database creation
--- CREATE TABLE ${hive_db_name}.Persons ( 
+--------------------------------------------------------------
--- PersonID int, 
+--------------------------------------------------------------
--- LastName varchar(255));
--- 
--- INSERT INTO ${hive_db_name}.Persons VALUES (1, "test_db_spyros_rec_111"); 
 
 DROP database IF EXISTS ${stats_db_name} CASCADE;
 CREATE database ${stats_db_name};

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10.sql

@@ -0,0 +1,21 @@
+------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------------
+-- Tables/views from external tables/views (Fundref, Country, CountyGDP, roarmap, rndexpediture)
+------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------------
+CREATE OR REPLACE VIEW  ${stats_db_name}.fundref AS SELECT * FROM ${external_stats_db_name}.fundref;
+CREATE OR REPLACE VIEW  ${stats_db_name}.country AS SELECT * FROM ${external_stats_db_name}.country;
+CREATE OR REPLACE VIEW  ${stats_db_name}.countrygdp AS SELECT * FROM ${external_stats_db_name}.countrygdp;
+CREATE OR REPLACE VIEW  ${stats_db_name}.roarmap AS SELECT * FROM ${external_stats_db_name}.roarmap;
+CREATE OR REPLACE VIEW  ${stats_db_name}.rndexpediture AS SELECT * FROM ${external_stats_db_name}.rndexpediture;
+CREATE OR REPLACE VIEW  ${stats_db_name}.context AS SELECT * FROM ${external_stats_db_name}.context;
+CREATE OR REPLACE VIEW  ${stats_db_name}.category AS SELECT * FROM ${external_stats_db_name}.category;
+CREATE OR REPLACE VIEW  ${stats_db_name}.concept AS SELECT * FROM ${external_stats_db_name}.concept;
+
+
+------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------------
+-- Creation date of the database
+------------------------------------------------------------------------------------------------
+------------------------------------------------------------------------------------------------
+create table ${stats_db_name}.creation_date as select date_format(current_date(), 'dd-MM-yyyy') as date;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10_1.sql

@@ -1,7 +0,0 @@
-----------------------------------------------------------------
-----------------------------------------------------------------
--- Organization table/view and Organization related tables/views
-----------------------------------------------------------------
-----------------------------------------------------------------
-DROP TABLE IF EXISTS ${stats_db_name}.organization;
-CREATE TABLE ${stats_db_name}.organization AS SELECT substr(o.id, 4) as id, o.legalname.value as name, o.country.classid as country from ${openaire_db_name}.organization o WHERE o.datainfo.deletedbyinference=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10_2.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.organization_datasources AS SELECT organization AS id, id AS datasource FROM ${stats_db_name}.datasource_organizations;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step10_3.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.organization_projects AS SELECT id AS project, organization as id FROM ${stats_db_name}.project_organizations;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step11.sql

@@ -1,10 +1,44 @@
-------------------------------------------------------------------------------------------------
+----------------------------------------------------------------
-------------------------------------------------------------------------------------------------
+----------------------------------------------------------------
--- Tables/views from external tables/views (Fundref, Country, CountyGDP, roarmap, rndexpediture)
+-- Post processing - Updates on main tables
-------------------------------------------------------------------------------------------------
+----------------------------------------------------------------
-------------------------------------------------------------------------------------------------
+----------------------------------------------------------------
-CREATE OR REPLACE VIEW  ${stats_db_name}.fundref AS SELECT * FROM ${external_stats_db_name}.fundref;
+
-CREATE OR REPLACE VIEW  ${stats_db_name}.country AS SELECT * FROM ${external_stats_db_name}.country;
+--Datasource temporary table updates
-CREATE OR REPLACE VIEW  ${stats_db_name}.countrygdp AS SELECT * FROM ${external_stats_db_name}.countrygdp;
+UPDATE ${stats_db_name}.datasource_tmp SET harvested='true' WHERE datasource_tmp.id IN (SELECT DISTINCT d.id FROM ${stats_db_name}.datasource_tmp d, ${stats_db_name}.result_datasources rd WHERE d.id=rd.datasource);
-CREATE OR REPLACE VIEW  ${stats_db_name}.roarmap AS SELECT * FROM ${external_stats_db_name}.roarmap;
+
-CREATE OR REPLACE VIEW  ${stats_db_name}.rndexpediture AS SELECT * FROM ${external_stats_db_name}.rndexpediture;
+-- Project temporary table update and final project table creation with final updates that can not be applied to ORC tables
+UPDATE ${stats_db_name}.project_tmp SET haspubs='yes' WHERE project_tmp.id IN (SELECT pr.id FROM ${stats_db_name}.project_results pr, ${stats_db_name}.result r WHERE pr.result=r.id AND r.type='publication');
+
+DROP TABLE IF EXISTS ${stats_db_name}.project;
+CREATE TABLE ${stats_db_name}.project stored as parquet as
+SELECT p.id , p.acronym, p.title, p.funder, p.funding_lvl0, p.funding_lvl1, p.funding_lvl2, p.ec39, p.type, p.startdate, p.enddate, p.start_year, p.end_year, p.duration, 
+CASE WHEN prr1.id IS NULL THEN 'no' ELSE 'yes' END AS haspubs, 
+CASE WHEN prr1.id IS NULL THEN 0 ELSE prr1.np END AS numpubs, 
+CASE WHEN prr2.id IS NULL THEN 0 ELSE prr2.daysForlastPub END AS daysforlastpub, 
+CASE WHEN prr2.id IS NULL THEN 0 ELSE prr2.dp END AS delayedpubs,
+p.callidentifier, p.code
+FROM ${stats_db_name}.project_tmp p 
+LEFT JOIN (SELECT pr.id, count(distinct pr.result) AS np
+        FROM ${stats_db_name}.project_results pr INNER JOIN ${stats_db_name}.result r ON pr.result=r.id 
+        WHERE r.type='publication' 
+        GROUP BY pr.id) AS prr1 on prr1.id = p.id
+LEFT JOIN (SELECT pp.id, max(datediff(to_date(r.date), to_date(pp.enddate)) ) AS daysForlastPub , count(distinct r.id) AS dp
+        FROM ${stats_db_name}.project_tmp pp, ${stats_db_name}.project_results pr, ${stats_db_name}.result r 
+        WHERE pp.id=pr.id AND pr.result=r.id AND r.type='publication' AND datediff(to_date(r.date), to_date(pp.enddate)) > 0 
+        GROUP BY pp.id) AS prr2
+        ON prr2.id = p.id;
+        
+-- Publication temporary table updates
+UPDATE ${stats_db_name}.publication_tmp SET delayed = 'yes' WHERE publication_tmp.id IN (SELECT distinct r.id FROM stats_wf_db_obs.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
+
+-- Dataset temporary table updates
+UPDATE ${stats_db_name}.dataset_tmp SET delayed = 'yes' WHERE dataset_tmp.id IN (SELECT distinct r.id FROM stats_wf_db_obs.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
+
+-- Software temporary table updates
+UPDATE ${stats_db_name}.software_tmp SET delayed = 'yes' WHERE software_tmp.id IN (SELECT distinct r.id FROM ${stats_db_name}.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
+
+-- Oherresearchproduct temporary table updates
+UPDATE ${stats_db_name}.otherresearchproduct_tmp SET delayed = 'yes' WHERE otherresearchproduct_tmp.id IN (SELECT distinct r.id FROM ${stats_db_name}.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);
+
+CREATE OR REPLACE VIEW ${stats_db_name}.project_results_publication AS SELECT result_projects.id AS result, result_projects.project AS project_results, result.date as resultdate, project.enddate as projectenddate, result_projects.daysfromend AS daysfromend FROM  ${stats_db_name}.result_projects, ${stats_db_name}.result, ${stats_db_name}.project WHERE result_projects.id=result.id AND result.type='publication' AND project.id=result_projects.project;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12.sql

@@ -0,0 +1,38 @@
+------------------------------------------------------------------------------------------------------
+-- Creating parquet tables from the updated temporary tables and removing unnecessary temporary tables
+------------------------------------------------------------------------------------------------------
+
+DROP TABLE IF EXISTS ${stats_db_name}.datasource;
+CREATE TABLE ${stats_db_name}.datasource stored AS parquet AS SELECT * FROM ${stats_db_name}.datasource_tmp;
+
+DROP TABLE IF EXISTS  ${stats_db_name}.publication;
+CREATE TABLE ${stats_db_name}.publication stored AS parquet AS SELECT * FROM ${stats_db_name}.publication_tmp;
+
+DROP TABLE IF EXISTS ${stats_db_name}.dataset;
+CREATE TABLE ${stats_db_name}.dataset stored AS parquet AS SELECT * FROM ${stats_db_name}.dataset_tmp;
+
+DROP TABLE IF EXISTS ${stats_db_name}.software;
+CREATE TABLE ${stats_db_name}.software stored AS parquet AS SELECT * FROM ${stats_db_name}.software_tmp;
+
+DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct;
+CREATE TABLE ${stats_db_name}.otherresearchproduct stored AS parquet AS SELECT * FROM ${stats_db_name}.otherresearchproduct_tmp;
+
+DROP TABLE ${stats_db_name}.project_tmp;
+DROP TABLE ${stats_db_name}.datasource_tmp;
+DROP TABLE ${stats_db_name}.publication_tmp;
+DROP TABLE ${stats_db_name}.dataset_tmp;
+DROP TABLE ${stats_db_name}.software_tmp;
+DROP TABLE ${stats_db_name}.otherresearchproduct_tmp;
+
+----------------------------------------------
+-- Re-creating views from final parquet tables
+---------------------------------------------
+
+-- Result
+CREATE OR REPLACE VIEW ${stats_db_name}.result AS SELECT *, bestlicence AS access_mode FROM ${stats_db_name}.publication UNION ALL SELECT *, bestlicence as access_mode FROM ${stats_db_name}.software UNION ALL SELECT *, bestlicence AS access_mode FROM ${stats_db_name}.dataset UNION ALL SELECT *, bestlicence AS access_mode FROM ${stats_db_name}.otherresearchproduct;
+
+
+-------------------------------------------------------------------------------
+-- To see with Antonis if the following is needed and where it should be placed
+-------------------------------------------------------------------------------
+CREATE TABLE ${stats_db_name}.numbers_country AS SELECT org.country AS country, count(distinct rd.datasource) AS datasources, count(distinct r.id) AS publications FROM ${stats_db_name}.result r, ${stats_db_name}.result_datasources rd, ${stats_db_name}.datasource d, ${stats_db_name}.datasource_organizations dor, ${stats_db_name}.organization org WHERE r.id=rd.id AND rd.datasource=d.id AND d.id=dor.id AND dor.organization=org.id AND r.type='publication' AND r.bestlicence='Open Access' GROUP BY org.country;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12_1.sql

@@ -1,6 +0,0 @@
-----------------------------
--- Post processing - Updates
-----------------------------
-
---Datasource temporary table updates
-UPDATE ${stats_db_name}.datasource_tmp set harvested ='true' WHERE datasource_tmp.id IN (SELECT DISTINCT d.id FROM ${stats_db_name}.datasource_tmp d, ${stats_db_name}.result_datasources rd where d.id=rd.datasource);

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12_2.sql

@@ -1,2 +0,0 @@
--- Project temporary table update and final project table creation with final updates that can not be applied to ORC tables
-UPDATE ${stats_db_name}.project_tmp SET haspubs='yes' WHERE project_tmp.id IN (SELECT pr.id FROM ${stats_db_name}.project_results pr, ${stats_db_name}.result r WHERE pr.result=r.id AND r.type='publication');

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12_3.sql

@@ -1,20 +0,0 @@
-DROP TABLE IF EXISTS ${stats_db_name}.project;
-
-CREATE TABLE ${stats_db_name}.project stored as parquet as
-SELECT p.id , p.acronym, p.title, p.funder, p.funding_lvl0, p.funding_lvl1, p.funding_lvl2, p.ec39, p.type, p.startdate, p.enddate, p.start_year, p.end_year, p.duration, 
-CASE WHEN prr1.id IS NULL THEN 'no' ELSE 'yes' END as haspubs, 
-CASE WHEN prr1.id IS NULL THEN 0 ELSE prr1.np END as numpubs, 
-CASE WHEN prr2.id IS NULL THEN 0 ELSE prr2.daysForlastPub END as daysforlastpub, 
-CASE WHEN prr2.id IS NULL THEN 0 ELSE prr2.dp END as delayedpubs,
-p.callidentifier, p.code
-FROM ${stats_db_name}.project_tmp p 
-LEFT JOIN (SELECT pr.id, count(distinct pr.result) AS np
-        FROM ${stats_db_name}.project_results pr INNER JOIN ${stats_db_name}.result r ON pr.result=r.id 
-        WHERE r.type='publication' 
-        GROUP BY pr.id) AS prr1 on prr1.id = p.id
-LEFT JOIN (SELECT pp.id, max(datediff(to_date(r.date), to_date(pp.enddate)) ) as daysForlastPub , count(distinct r.id) as dp
-        FROM ${stats_db_name}.project_tmp pp, ${stats_db_name}.project_results pr, ${stats_db_name}.result r 
-        WHERE pp.id=pr.id AND pr.result=r.id AND r.type='publication' AND datediff(to_date(r.date), to_date(pp.enddate)) > 0 
-        GROUP BY pp.id) AS prr2
-        on prr2.id = p.id;
-        

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12_4.sql

@@ -1,2 +0,0 @@
--- Publication temporary table updates
-UPDATE ${stats_db_name}.publication_tmp SET delayed = 'yes' WHERE publication_tmp.id IN (SELECT distinct r.id FROM stats_wf_db_obs.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12_5.sql

@@ -1,2 +0,0 @@
--- Dataset temporary table updates
-UPDATE ${stats_db_name}.dataset_tmp SET delayed = 'yes' WHERE dataset_tmp.id IN (SELECT distinct r.id FROM stats_wf_db_obs.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12_6.sql

@@ -1,2 +0,0 @@
--- Software temporary table updates
-UPDATE ${stats_db_name}.software_tmp SET delayed = 'yes' WHERE software_tmp.id IN (SELECT distinct r.id FROM ${stats_db_name}.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12_7.sql

@@ -1,2 +0,0 @@
--- Oherresearchproduct temporary table updates
-UPDATE ${stats_db_name}.otherresearchproduct_tmp SET delayed = 'yes' WHERE otherresearchproduct_tmp.id IN (SELECT distinct r.id FROM ${stats_db_name}.result r, ${stats_db_name}.project_results pr, ${stats_db_name}.project_tmp p WHERE r.id=pr.result AND pr.id=p.id AND to_date(r.date)-to_date(p.enddate) > 0);

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12_8.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.project_results_publication AS SELECT result_projects.id AS result, result_projects.project AS project_results, result.date as resultdate, project.enddate as projectenddate, result_projects.daysfromend as daysfromend FROM  ${stats_db_name}.result_projects, ${stats_db_name}.result, ${stats_db_name}.project WHERE result_projects.id=result.id and result.type='publication' and project.id=result_projects.project;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step13.sql

@@ -1,26 +1,59 @@
-------------------------------------------------------------------------------------------------------
+------------------------------------------------------
--- Creating parquet tables from the updated temporary tables and removing unnecessary temporary tables
+------------------------------------------------------
-------------------------------------------------------------------------------------------------------
+-- Additional relations
+--
+-- Sources related tables/views
+------------------------------------------------------
+------------------------------------------------------
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_sources as 
+SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
+FROM (
+    SELECT  substr(p.id, 4) as id, substr(datasource, 4) as datasource 
+from ${openaire_db_name}.publication p lateral view explode(p.collectedfrom.key) c as datasource) p 
+LEFT OUTER JOIN
+(
+    SELECT substr(d.id, 4) id 
+    from ${openaire_db_name}.datasource d 
+    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
 
-DROP TABLE IF EXISTS ${stats_db_name}.datasource;
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_sources as 
-CREATE TABLE ${stats_db_name}.datasource stored as parquet as select * from ${stats_db_name}.datasource_tmp;
+SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
+FROM (
+    SELECT  substr(p.id, 4) as id, substr(datasource, 4) as datasource 
+from ${openaire_db_name}.dataset p lateral view explode(p.collectedfrom.key) c as datasource) p 
+LEFT OUTER JOIN
+(
+    SELECT substr(d.id, 4) id 
+    from ${openaire_db_name}.datasource d 
+    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
     
-DROP TABLE IF EXISTS  ${stats_db_name}.publication;
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_sources as 
-CREATE TABLE ${stats_db_name}.publication stored as parquet as select * from ${stats_db_name}.publication_tmp;
+SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
+FROM (
+    SELECT  substr(p.id, 4) as id, substr(datasource, 4) as datasource 
+from ${openaire_db_name}.software p lateral view explode(p.collectedfrom.key) c as datasource) p 
+LEFT OUTER JOIN
+(
+    SELECT substr(d.id, 4) id 
+    from ${openaire_db_name}.datasource d 
+    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
     
-DROP TABLE IF EXISTS ${stats_db_name}.dataset;
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_sources as 
-CREATE TABLE ${stats_db_name}.dataset stored as parquet as select * from ${stats_db_name}.dataset_tmp;
+SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource 
-
+FROM (
-DROP TABLE IF EXISTS ${stats_db_name}.software;
+    SELECT  substr(p.id, 4) as id, substr(datasource, 4) as datasource 
-CREATE TABLE ${stats_db_name}.software stored as parquet as select * from ${stats_db_name}.software_tmp;
+from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.collectedfrom.key) c as datasource) p 
-
+LEFT OUTER JOIN
-DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct;
+(
-CREATE TABLE ${stats_db_name}.otherresearchproduct stored as parquet as select * from ${stats_db_name}.otherresearchproduct_tmp;
+    SELECT substr(d.id, 4) id 
-
+    from ${openaire_db_name}.datasource d 
-DROP TABLE ${stats_db_name}.project_tmp;
+    WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
-DROP TABLE ${stats_db_name}.datasource_tmp;
-DROP TABLE ${stats_db_name}.publication_tmp;
-DROP TABLE ${stats_db_name}.dataset_tmp;
-DROP TABLE ${stats_db_name}.software_tmp;
-DROP TABLE ${stats_db_name}.otherresearchproduct_tmp;
     
+CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_sources AS
+SELECT * FROM ${stats_db_name}.publication_sources
+UNION ALL
+SELECT * FROM ${stats_db_name}.dataset_sources
+UNION ALL
+SELECT * FROM ${stats_db_name}.software_sources
+UNION ALL
+SELECT * FROM ${stats_db_name}.otherresearchproduct_sources;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step14.sql

@@ -1,7 +1,49 @@
-----------------------------------------------
+------------------------------------------------------
--- Re-creating views from final parquet tables
+------------------------------------------------------
----------------------------------------------
+-- Additional relations
+--
+-- Licences related tables/views
+------------------------------------------------------
+------------------------------------------------------
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_licenses AS
+SELECT substr(p.id, 4) as id, licenses.value as type 
+from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.license) instances as licenses
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
 
--- Result
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_licenses AS
-CREATE OR REPLACE VIEW ${stats_db_name}.result as SELECT *, bestlicence as access_mode FROM ${stats_db_name}.publication UNION ALL SELECT *, bestlicence as access_mode FROM ${stats_db_name}.software UNION ALL SELECT *, bestlicence as access_mode FROM ${stats_db_name}.dataset UNION ALL SELECT *, bestlicence as access_mode FROM ${stats_db_name}.otherresearchproduct;
+SELECT substr(p.id, 4) as id, licenses.value as type 
+from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.license) instances as licenses
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
 
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_licenses AS
+SELECT substr(p.id, 4) as id, licenses.value as type 
+from ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.license) instances as licenses
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_licenses AS
+SELECT substr(p.id, 4) as id, licenses.value as type 
+from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.license) instances as licenses
+where licenses.value is not null and licenses.value != '' and p.datainfo.deletedbyinference=false;
+
+CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_licenses AS
+SELECT * FROM ${stats_db_name}.publication_licenses
+UNION ALL
+SELECT * FROM ${stats_db_name}.dataset_licenses
+UNION ALL
+SELECT * FROM ${stats_db_name}.software_licenses
+UNION ALL
+SELECT * FROM ${stats_db_name}.otherresearchproduct_licenses;
+
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization_pids AS 
+select substr(o.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid 
+from ${openaire_db_name}.organization o lateral view explode(o.pid) pids as ppid;
+
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization_sources as 
+SELECT o.id, case when d.id is null then 'other' else o.datasource end as datasource 
+FROM (
+    SELECT  substr(o.id, 4) as id, substr(instances.instance.key, 4) as datasource 
+    from ${openaire_db_name}.organization o lateral view explode(o.collectedfrom) instances as instance) o 
+    LEFT OUTER JOIN (
+        SELECT substr(d.id, 4) id 
+        from ${openaire_db_name}.datasource d 
+        WHERE d.datainfo.deletedbyinference=false) d on o.datasource = d.id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step15.sql

@@ -0,0 +1,36 @@
+------------------------------------------------------
+------------------------------------------------------
+-- Additional relations
+--
+-- Refereed related tables/views
+------------------------------------------------------
+------------------------------------------------------
+
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.publication_refereed as
+select substr(r.id, 4) as id, inst.refereed.classname as refereed
+from ${openaire_db_name}.publication r lateral view explode(r.instance) instances as inst
+where r.datainfo.deletedbyinference=false;
+
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.dataset_refereed as
+select substr(r.id, 4) as id, inst.refereed.classname as refereed
+from ${openaire_db_name}.dataset r lateral view explode(r.instance) instances as inst
+where r.datainfo.deletedbyinference=false;
+
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.software_refereed as
+select substr(r.id, 4) as id, inst.refereed.classname as refereed
+from ${openaire_db_name}.software r lateral view explode(r.instance) instances as inst
+where r.datainfo.deletedbyinference=false;
+
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.otherresearchproduct_refereed as
+select substr(r.id, 4) as id, inst.refereed.classname as refereed
+from ${openaire_db_name}.otherresearchproduct r lateral view explode(r.instance) instances as inst
+where r.datainfo.deletedbyinference=false;
+
+CREATE VIEW IF NOT EXISTS ${stats_db_name}.result_refereed as
+select * from ${stats_db_name}.publication_refereed
+union all
+select * from ${stats_db_name}.dataset_refereed
+union all
+select * from ${stats_db_name}.software_refereed
+union all
+select * from ${stats_db_name}.otherresearchproduct_refereed;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16.sql

@@ -0,0 +1,80 @@
+----------------------------------------------------
+-- Shortcuts for various definitions in stats db ---
+----------------------------------------------------
+
+-- Peer reviewed:
+-- Results that have been collected from Crossref
+create table ${stats_db_name}.result_peerreviewed as
+with peer_reviewed as (
+    select distinct r.id as id
+    from ${stats_db_name}.result r
+    join ${stats_db_name}.result_sources rs on rs.id=r.id
+    join ${stats_db_name}.datasource d on d.id=rs.datasource
+    where d.name='Crossref')
+select distinct peer_reviewed.id as id, true as peer_reviewed
+from peer_reviewed
+union all
+select distinct r.id as id, false as peer_reviewed
+from ${stats_db_name}.result r
+left outer join peer_reviewed pr on pr.id=r.id
+where pr.id is null;
+
+-- Green OA:
+-- OA results that are hosted by an Institutional repository and have NOT been harvested from a DOAJ journal.
+create table ${stats_db_name}.result_greenoa as
+with result_green as (
+    select distinct r.id as id
+    from ${stats_db_name}.result r
+    join ${stats_db_name}.result_datasources rd on rd.id=r.id
+    join ${stats_db_name}.datasource d on d.id=rd.datasource
+    left outer join (
+        select rd.id from ${stats_db_name}.result_datasources rd
+        join ${stats_db_name}.datasource d on rd.datasource=d.id
+        join ${stats_db_name}.datasource_sources sds on sds.id=d.id
+        join ${stats_db_name}.datasource sd on sd.id=sds.datasource
+        where sd.name='DOAJ-ARTICLES'
+    ) as doaj on doaj.id=r.id
+    where r.bestlicence in ('Open Access', 'Open Source') and d.type='Institutional Repository' and doaj.id is null)
+select distinct result_green.id, true as green
+from result_green
+union all
+select distinct r.id as id, false as green
+from ${stats_db_name}.result r
+left outer join result_green rg on rg.id=r.id
+where rg.id  is null;
+
+-- GOLD OA:
+-- OA results that have been harvested from a DOAJ journal.
+create table ${stats_db_name}.result_gold as
+with result_gold as (
+    select distinct r.id as id
+    from ${stats_db_name}.result r
+    join ${stats_db_name}.result_datasources rd on rd.id=r.id
+    join ${stats_db_name}.datasource d on d.id=rd.datasource
+    join ${stats_db_name}.datasource_sources sds on sds.id=d.id
+    join ${stats_db_name}.datasource sd on sd.id=sds.datasource
+    where r.type='publication' and r.bestlicence='Open Access' and sd.name='DOAJ-Articles')
+select distinct result_gold.id, true as gold
+from result_gold
+union all
+select distinct r.id, false as gold
+from ${stats_db_name}.result r
+where r.id not in (select id from result_gold);
+
+-- shortcut result-country through the organization affiliation
+create table ${stats_db_name}.result_affiliated_country as
+select r.id as id, o.country as country
+from ${stats_db_name}.result r
+join ${stats_db_name}.result_organization ro on ro.id=r.id
+join ${stats_db_name}.organization o on o.id=ro.organization
+where o.country is not null and o.country!='';
+
+-- shortcut result-country through datasource of deposition
+create table ${stats_db_name}.result_deposited_country as
+select r.id as id, o.country as country
+from ${stats_db_name}.result r
+join ${stats_db_name}.result_datasources rd on rd.id=r.id
+join ${stats_db_name}.datasource d on d.id=rd.datasource
+join ${stats_db_name}.datasource_organizations dor on dor.id=d.id
+join ${stats_db_name}.organization o on o.id=dor.organization
+where o.country is not null and o.country!='';

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_5.sql

@@ -0,0 +1,55 @@
+-- replace the creation of the result view to include the boolean fields from the previous tables (green, gold,
+-- peer reviewed)
+drop table if exists ${stats_db_name}.result_tmp;
+CREATE TABLE ${stats_db_name}.result_tmp (
+    id STRING,
+    title STRING,
+    publisher STRING,
+    journal STRING,
+    `date` STRING,
+    `year` INT,
+    bestlicence STRING,
+    access_mode STRING,
+    embargo_end_date STRING,
+    delayed BOOLEAN,
+    authors INT,
+    source STRING,
+    abstract BOOLEAN,
+    type STRING ,
+    peer_reviewed BOOLEAN,
+    green BOOLEAN,
+    gold BOOLEAN)
+clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
+
+insert into ${stats_db_name}.result_tmp
+select r.id, r.title, r.publisher, r.journal, r.`date`, date_format(r.`date`, 'yyyy'), r.bestlicence, r.bestlicence, r.embargo_end_date, r.delayed, r.authors, r.source, r.abstract, r.type, pr.peer_reviewed, green.green, gold.gold
+FROM ${stats_db_name}.publication r
+LEFT OUTER JOIN ${stats_db_name}.result_peerreviewed pr on pr.id=r.id
+LEFT OUTER JOIN ${stats_db_name}.result_greenoa green on green.id=r.id
+LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
+
+insert into ${stats_db_name}.result_tmp
+select r.id, r.title, r.publisher, r.journal, r.`date`, date_format(r.`date`, 'yyyy'), r.bestlicence, r.bestlicence, r.embargo_end_date, r.delayed, r.authors, r.source, r.abstract, r.type, pr.peer_reviewed, green.green, gold.gold
+FROM ${stats_db_name}.dataset r
+LEFT OUTER JOIN ${stats_db_name}.result_peerreviewed pr on pr.id=r.id
+LEFT OUTER JOIN ${stats_db_name}.result_greenoa green on green.id=r.id
+LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
+
+insert into ${stats_db_name}.result_tmp
+select r.id, r.title, r.publisher, r.journal, r.`date`, date_format(r.`date`, 'yyyy'), r.bestlicence, r.bestlicence, r.embargo_end_date, r.delayed, r.authors, r.source, r.abstract, r.type, pr.peer_reviewed, green.green, gold.gold
+FROM ${stats_db_name}.software r
+LEFT OUTER JOIN ${stats_db_name}.result_peerreviewed pr on pr.id=r.id
+LEFT OUTER JOIN ${stats_db_name}.result_greenoa green on green.id=r.id
+LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
+
+insert into ${stats_db_name}.result_tmp
+select r.id, r.title, r.publisher, r.journal, r.`date`, date_format(r.`date`, 'yyyy'), r.bestlicence, r.bestlicence, r.embargo_end_date, r.delayed, r.authors, r.source, r.abstract, r.type, pr.peer_reviewed, green.green, gold.gold
+FROM ${stats_db_name}.otherresearchproduct r
+LEFT OUTER JOIN ${stats_db_name}.result_peerreviewed pr on pr.id=r.id
+LEFT OUTER JOIN ${stats_db_name}.result_greenoa green on green.id=r.id
+LEFT OUTER JOIN ${stats_db_name}.result_gold gold on gold.id=r.id;
+
+drop table if exists ${stats_db_name}.result;
+drop view if exists ${stats_db_name}.result;
+create table ${stats_db_name}.result stored as parquet as select * from ${stats_db_name}.result_tmp;
+drop table ${stats_db_name}.result_tmp;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step17.sql

@@ -0,0 +1,163 @@
+------------------------------------------------------
+------------------------------------------------------
+-- Shadow schema table exchange
+------------------------------------------------------
+------------------------------------------------------
+
+-- Dropping old views
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.country;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.countrygdp;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_citations;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_classifications;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_concepts;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_datasources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_languages;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_licenses;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_oids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_pids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_sources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.dataset_topics;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_languages;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_oids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_organizations;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.datasource_results;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.fundref;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.numbers_country;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_datasources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.organization_projects;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_citations;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_classifications;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_concepts;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_datasources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_languages;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_licenses;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_oids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_pids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_sources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.otherresearchproduct_topics;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.project;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_oids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_organizations;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.project_results;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_citations;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_classifications;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_concepts;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_datasources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_languages;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_licenses;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_oids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_pids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_sources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.publication_topics;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_citations;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_classifications;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_concepts;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_datasources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_languages;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_licenses;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_oids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_organization;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_pids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_projects;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_sources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.result_topics;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.rndexpediture;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.roarmap;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_citations;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_classifications;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_concepts;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_datasources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_languages;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_licenses;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_oids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_pids;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_sources;
+DROP VIEW IF EXISTS ${stats_db_shadow_name}.software_topics;
+
+
+-- Creating the shadow database, in case it doesn't exist
+CREATE database IF NOT EXISTS ${stats_db_shadow_name};
+
+-- Creating new views
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.country AS SELECT * FROM ${stats_db_name}.country;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.countrygdp AS SELECT * FROM ${stats_db_name}.countrygdp;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset AS SELECT * FROM ${stats_db_name}.dataset;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_citations AS SELECT * FROM ${stats_db_name}.dataset_citations;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_classifications AS SELECT * FROM ${stats_db_name}.dataset_classifications;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_concepts AS SELECT * FROM ${stats_db_name}.dataset_concepts;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_datasources AS SELECT * FROM ${stats_db_name}.dataset_datasources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_languages AS SELECT * FROM ${stats_db_name}.dataset_languages;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_licenses AS SELECT * FROM ${stats_db_name}.dataset_licenses;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_oids AS SELECT * FROM ${stats_db_name}.dataset_oids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_pids AS SELECT * FROM ${stats_db_name}.dataset_pids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_sources AS SELECT * FROM ${stats_db_name}.dataset_sources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.dataset_topics AS SELECT * FROM ${stats_db_name}.dataset_topics;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource AS SELECT * FROM ${stats_db_name}.datasource;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_languages AS SELECT * FROM ${stats_db_name}.datasource_languages;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_oids AS SELECT * FROM ${stats_db_name}.datasource_oids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_organizations AS SELECT * FROM ${stats_db_name}.datasource_organizations;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.datasource_results AS SELECT * FROM ${stats_db_name}.datasource_results;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.fundref AS SELECT * FROM ${stats_db_name}.fundref;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.numbers_country AS SELECT * FROM ${stats_db_name}.numbers_country;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization AS SELECT * FROM ${stats_db_name}.organization;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_datasources AS SELECT * FROM ${stats_db_name}.organization_datasources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.organization_projects AS SELECT * FROM ${stats_db_name}.organization_projects;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct AS SELECT * FROM ${stats_db_name}.otherresearchproduct;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_citations AS SELECT * FROM ${stats_db_name}.otherresearchproduct_citations;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_classifications AS SELECT * FROM ${stats_db_name}.otherresearchproduct_classifications;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_concepts AS SELECT * FROM ${stats_db_name}.otherresearchproduct_concepts;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_datasources AS SELECT * FROM ${stats_db_name}.otherresearchproduct_datasources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_languages AS SELECT * FROM ${stats_db_name}.otherresearchproduct_languages;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_licenses AS SELECT * FROM ${stats_db_name}.otherresearchproduct_licenses;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_oids AS SELECT * FROM ${stats_db_name}.otherresearchproduct_oids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_pids AS SELECT * FROM ${stats_db_name}.otherresearchproduct_pids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_sources AS SELECT * FROM ${stats_db_name}.otherresearchproduct_sources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.otherresearchproduct_topics AS SELECT * FROM ${stats_db_name}.otherresearchproduct_topics;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project AS SELECT * FROM ${stats_db_name}.project;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_oids AS SELECT * FROM ${stats_db_name}.project_oids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_organizations AS SELECT * FROM ${stats_db_name}.project_organizations;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.project_results AS SELECT * FROM ${stats_db_name}.project_results;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication AS SELECT * FROM ${stats_db_name}.publication;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_citations AS SELECT * FROM ${stats_db_name}.publication_citations;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_classifications AS SELECT * FROM ${stats_db_name}.publication_classifications;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_concepts AS SELECT * FROM ${stats_db_name}.publication_concepts;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_datasources AS SELECT * FROM ${stats_db_name}.publication_datasources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_languages AS SELECT * FROM ${stats_db_name}.publication_languages;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_licenses AS SELECT * FROM ${stats_db_name}.publication_licenses;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_oids AS SELECT * FROM ${stats_db_name}.publication_oids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_pids AS SELECT * FROM ${stats_db_name}.publication_pids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_sources AS SELECT * FROM ${stats_db_name}.publication_sources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.publication_topics AS SELECT * FROM ${stats_db_name}.publication_topics;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result AS SELECT * FROM ${stats_db_name}.result;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_citations AS SELECT * FROM ${stats_db_name}.result_citations;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_classifications AS SELECT * FROM ${stats_db_name}.result_classifications;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_concepts AS SELECT * FROM ${stats_db_name}.result_concepts;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_datasources AS SELECT * FROM ${stats_db_name}.result_datasources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_languages AS SELECT * FROM ${stats_db_name}.result_languages;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_licenses AS SELECT * FROM ${stats_db_name}.result_licenses;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_oids AS SELECT * FROM ${stats_db_name}.result_oids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_organization AS SELECT * FROM ${stats_db_name}.result_organization;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_pids AS SELECT * FROM ${stats_db_name}.result_pids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_projects AS SELECT * FROM ${stats_db_name}.result_projects;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_sources AS SELECT * FROM ${stats_db_name}.result_sources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.result_topics AS SELECT * FROM ${stats_db_name}.result_topics;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.rndexpediture AS SELECT * FROM ${stats_db_name}.rndexpediture;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.roarmap AS SELECT * FROM ${stats_db_name}.roarmap;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software AS SELECT * FROM ${stats_db_name}.software;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_citations AS SELECT * FROM ${stats_db_name}.software_citations;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_classifications AS SELECT * FROM ${stats_db_name}.software_classifications;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_concepts AS SELECT * FROM ${stats_db_name}.software_concepts;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_datasources AS SELECT * FROM ${stats_db_name}.software_datasources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_languages AS SELECT * FROM ${stats_db_name}.software_languages;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_licenses AS SELECT * FROM ${stats_db_name}.software_licenses;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_oids AS SELECT * FROM ${stats_db_name}.software_oids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_pids AS SELECT * FROM ${stats_db_name}.software_pids;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_sources AS SELECT * FROM ${stats_db_name}.software_sources;
+CREATE VIEW IF NOT EXISTS ${stats_db_shadow_name}.software_topics AS SELECT * FROM ${stats_db_name}.software_topics;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step18.sql

@@ -0,0 +1,81 @@
+------------------------------------------------------
+------------------------------------------------------
+-- Impala table statistics - Needed to make the tables
+-- visible for impala
+------------------------------------------------------
+------------------------------------------------------
+
+COMPUTE STATS country;
+COMPUTE STATS countrygdp;
+COMPUTE STATS dataset;
+COMPUTE STATS dataset_citations;
+COMPUTE STATS dataset_classifications;
+COMPUTE STATS dataset_concepts;
+COMPUTE STATS dataset_datasources;
+COMPUTE STATS dataset_languages;
+COMPUTE STATS dataset_oids;
+COMPUTE STATS dataset_pids;
+COMPUTE STATS dataset_sources;
+COMPUTE STATS dataset_topics;
+COMPUTE STATS datasource;
+COMPUTE STATS datasource_languages;
+COMPUTE STATS datasource_oids;
+COMPUTE STATS datasource_organizations;
+COMPUTE STATS datasource_results;
+COMPUTE STATS fundref;
+COMPUTE STATS numbers_country;
+COMPUTE STATS organization;
+COMPUTE STATS organization_datasources;
+COMPUTE STATS organization_projects;
+COMPUTE STATS otherresearchproduct;
+COMPUTE STATS otherresearchproduct_citations;
+COMPUTE STATS otherresearchproduct_classifications;
+COMPUTE STATS otherresearchproduct_concepts;
+COMPUTE STATS otherresearchproduct_datasources;
+COMPUTE STATS otherresearchproduct_languages;
+COMPUTE STATS otherresearchproduct_licenses;
+COMPUTE STATS otherresearchproduct_oids;
+COMPUTE STATS otherresearchproduct_pids;
+COMPUTE STATS otherresearchproduct_sources;
+COMPUTE STATS otherresearchproduct_topics;
+COMPUTE STATS project;
+COMPUTE STATS project_oids;
+COMPUTE STATS project_organizations;
+COMPUTE STATS project_results;
+COMPUTE STATS publication;
+COMPUTE STATS publication_citations;
+COMPUTE STATS publication_classifications;
+COMPUTE STATS publication_concepts;
+COMPUTE STATS publication_datasources;
+COMPUTE STATS publication_languages;
+COMPUTE STATS publication_licenses;
+COMPUTE STATS publication_oids;
+COMPUTE STATS publication_pids;
+COMPUTE STATS publication_sources;
+COMPUTE STATS publication_topics;
+COMPUTE STATS result;
+COMPUTE STATS result_citations;
+COMPUTE STATS result_classifications;
+COMPUTE STATS result_concepts;
+COMPUTE STATS result_datasources;
+COMPUTE STATS result_languages;
+COMPUTE STATS result_licenses;
+COMPUTE STATS result_oids;
+COMPUTE STATS result_organization;
+COMPUTE STATS result_pids;
+COMPUTE STATS result_projects;
+COMPUTE STATS result_sources;
+COMPUTE STATS result_topics;
+COMPUTE STATS rndexpediture;
+COMPUTE STATS roarmap;
+COMPUTE STATS software;
+COMPUTE STATS software_citations;
+COMPUTE STATS software_classifications;
+COMPUTE STATS software_concepts;
+COMPUTE STATS software_datasources;
+COMPUTE STATS software_languages;
+COMPUTE STATS software_licenses;
+COMPUTE STATS software_oids;
+COMPUTE STATS software_pids;
+COMPUTE STATS software_sources;
+COMPUTE STATS software_topics;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2.sql

@@ -0,0 +1,35 @@
+--------------------------------------------------------------
+--------------------------------------------------------------
+-- Publication table/view and Publication related tables/views
+--------------------------------------------------------------
+--------------------------------------------------------------
+
+-- Publication temporary table
+DROP TABLE IF EXISTS ${stats_db_name}.publication_tmp;
+
+CREATE TABLE ${stats_db_name}.publication_tmp (id STRING,   title STRING,   publisher STRING,   journal STRING,   date STRING,   year STRING,   bestlicence STRING,   embargo_end_date STRING,   delayed BOOLEAN,   authors INT,   source STRING,   abstract BOOLEAN,   type STRING )  clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');
+
+INSERT INTO ${stats_db_name}.publication_tmp SELECT substr(p.id, 4) as id, p.title[0].value as title, p.publisher.value as publisher, p.journal.name as journal , 
+p.dateofacceptance.value as date, date_format(p.dateofacceptance.value,'yyyy') as year, p.bestaccessright.classname as bestlicence,
+p.embargoenddate.value as embargo_end_date, false as delayed, size(p.author) as authors , concat_ws('\u003B',p.source.value) as source,
+case when size(p.description) > 0 then true else false end as abstract,
+'publication' as type
+from ${openaire_db_name}.publication p
+where p.datainfo.deletedbyinference=false;
+
+CREATE TABLE ${stats_db_name}.publication_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;
+
+CREATE TABLE ${stats_db_name}.publication_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.publication p LATERAL VIEW explode(p.context) contexts as context;
+
+CREATE TABLE ${stats_db_name}.publication_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT  substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource from ${openaire_db_name}.publication p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN (SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+
+CREATE TABLE ${stats_db_name}.publication_languages AS select substr(p.id, 4) as id, p.language.classname as language FROM ${openaire_db_name}.publication p;
+
+CREATE TABLE ${stats_db_name}.publication_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.originalid) oids AS ids;
+
+CREATE TABLE ${stats_db_name}.publication_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value as pid FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.pid) pids AS ppid;
+
+CREATE TABLE ${stats_db_name}.publication_topics as select substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS TYPE, subjects.subject.value AS topic FROM ${openaire_db_name}.publication p LATERAL VIEW explode(p.subject) subjects AS subject;
+
+-- Publication_citations
+CREATE TABLE ${stats_db_name}.publication_citations AS SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.publication p lateral view explode(p.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_1.sql

@@ -1,10 +0,0 @@
---------------------------------------------------------------
---------------------------------------------------------------
--- 2. Publication table/view and Publication related tables/views
---------------------------------------------------------------
---------------------------------------------------------------
-
--- Publication temporary table
-DROP TABLE IF EXISTS ${stats_db_name}.publication_tmp;
-
-CREATE TABLE ${stats_db_name}.publication_tmp (id STRING,   title STRING,   publisher STRING,   journal STRING,   date STRING,   year STRING,   bestlicence STRING,   embargo_end_date STRING,   delayed BOOLEAN,   authors INT,   source STRING,   abstract BOOLEAN,   type STRING )  clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_2.sql

@@ -1,19 +0,0 @@
--- The following throws the following exception on CRN HUE Hive:
--- Error while compiling statement: FAILED: SemanticException [Error 10011]: Line 2:34 Invalid function 'date_format'
--- But runs OK on OCEAN HUE Hive
-
-INSERT INTO ${stats_db_name}.publication_tmp SELECT substr(p.id, 4) as id, p.title[0].value as title, p.publisher.value as publisher, p.journal.name as journal , 
-p.dateofacceptance.value as date, date_format(p.dateofacceptance.value,'yyyy') as year, p.bestaccessright.classname as bestlicence,
-p.embargoenddate.value as embargo_end_date, false as delayed, size(p.author) as authors , concat_ws('\u003B',p.source.value) as source,
-case when size(p.description) > 0 then true else false end as abstract,
-'publication' as type
-from ${openaire_db_name}.publication p
-where p.datainfo.deletedbyinference=false;
-
--- INSERT INTO ${hive_db_name}.publication_tmp SELECT substr(p.id, 4) as id, p.title[0].value as title, p.publisher.value as publisher, p.journal.name as journal,
--- p.dateofacceptance.value as date, date_format(p.dateofacceptance.value,'yyyy') as year, p.bestaccessright.classname as bestlicence,
--- p.embargoenddate.value as embargo_end_date, false as delayed, size(p.author) as authors , concat_ws('\u003B',p.source.value) as source,
--- case when size(p.description) > 0 then true else false end as abstract,
--- 'publication' as type
--- from openaire.publication p
--- where p.datainfo.deletedbyinference=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_3.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.publication_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.publication p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_4.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.publication_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.publication p LATERAL VIEW explode(p.context) contexts as context;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_5.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.publication_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT  substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource from ${openaire_db_name}.publication p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN (SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_6.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.publication_languages AS select substr(p.id, 4) as id, p.language.classname as language from ${openaire_db_name}.publication p;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_7.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.publication_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.publication p LATERAL VIEW explode(p.originalid) oids as ids;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_8.sql

@@ -1 +0,0 @@
-create table ${stats_db_name}.publication_pids as select substr(p.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.publication p lateral view explode(p.pid) pids as ppid;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step2_9.sql

@@ -1 +0,0 @@
-create table ${stats_db_name}.publication_topics as select substr(p.id, 4) as id, subjects.subject.qualifier.classname as type, subjects.subject.value as topic from ${openaire_db_name}.publication p lateral view explode(p.subject) subjects as subject;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step3.sql

@@ -1,2 +1,36 @@
--- 3. Publication_citations
+------------------------------------------------------
-CREATE TABLE ${stats_db_name}.publication_citations AS SELECT substr(p.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.publication p lateral view explode(p.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
+------------------------------------------------------
+-- Dataset table/view and Dataset related tables/views
+------------------------------------------------------
+------------------------------------------------------
+
+-- Dataset temporary table supporting updates
+DROP TABLE IF EXISTS ${stats_db_name}.dataset_tmp;
+CREATE TABLE ${stats_db_name}.dataset_tmp (id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN,   authors INT,   source STRING,   abstract BOOLEAN,   type STRING ) clustered by (id) into 100 buckets stored AS orc tblproperties('transactional'='true');
+
+INSERT INTO ${stats_db_name}.dataset_tmp SELECT substr(d.id, 4) AS id, d.title[0].value AS title, d.publisher.value AS publisher, cast(null AS string) AS journal, 
+d.dateofacceptance.value as date, date_format(d.dateofacceptance.value,'yyyy') AS year, d.bestaccessright.classname AS bestlicence,
+d.embargoenddate.value AS embargo_end_date, false AS delayed, size(d.author) AS authors , concat_ws('\u003B',d.source.value) AS source,
+ CASE WHEN SIZE(d.description) > 0 THEN TRUE ELSE FALSE end AS abstract,
+'dataset' AS type
+FROM ${openaire_db_name}.dataset d
+WHERE d.datainfo.deletedbyinference=FALSE;
+
+-- Dataset_citations
+CREATE TABLE ${stats_db_name}.dataset_citations AS SELECT substr(d.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS result FROM ${openaire_db_name}.dataset d  LATERAL VIEW explode(d.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
+
+CREATE TABLE ${stats_db_name}.dataset_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
+
+CREATE TABLE ${stats_db_name}.dataset_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.context) contexts as context;
+
+CREATE TABLE ${stats_db_name}.dataset_datasources AS SELECT p.id, case when d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource FROM (SELECT  substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) AS datasource 
+FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance) instances AS instance) p LEFT OUTER JOIN
+(SELECT substr(d.id, 4) id FROM ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d ON p.datasource = d.id;
+
+CREATE TABLE ${stats_db_name}.dataset_languages AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.dataset p;
+
+CREATE TABLE ${stats_db_name}.dataset_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.originalid) oids AS ids;
+
+CREATE TABLE ${stats_db_name}.dataset_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.pid) pids AS ppid;
+
+CREATE TABLE ${stats_db_name}.dataset_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.dataset p LATERAL VIEW explode(p.subject) subjects AS subject;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4.sql

@@ -0,0 +1,36 @@
+--------------------------------------------------------
+--------------------------------------------------------
+-- Software table/view and Software related tables/views
+--------------------------------------------------------
+--------------------------------------------------------
+
+-- Software temporary table supporting updates
+DROP TABLE IF EXISTS ${stats_db_name}.software_tmp;
+CREATE TABLE ${stats_db_name}.software_tmp (id STRING,   title STRING,   publisher STRING,   journal STRING,   date STRING,   year STRING,   bestlicence STRING,   embargo_end_date STRING,   delayed BOOLEAN,   authors INT,   source STRING,   abstract BOOLEAN,   type STRING )  clustered by (id) INTO 100 buckets stored AS orc tblproperties('transactional'='true');
+
+INSERT INTO ${stats_db_name}.software_tmp SELECT substr(s.id, 4) as id, s.title[0].value AS title, s.publisher.value AS publisher, CAST(NULL AS string) AS journal, 
+s.dateofacceptance.value AS DATE, date_format(s.dateofacceptance.value,'yyyy') AS YEAR, s.bestaccessright.classname AS bestlicence,
+s.embargoenddate.value AS embargo_end_date, FALSE AS delayed, SIZE(s.author) AS authors , concat_ws('\u003B',s.source.value) AS source,
+ CASE WHEN SIZE(s.description) > 0 THEN TRUE ELSE FALSE END AS abstract,
+'software' as type
+from ${openaire_db_name}.software s
+where s.datainfo.deletedbyinference=false;
+
+-- Software_citations
+CREATE TABLE ${stats_db_name}.software_citations AS SELECT substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT FROM ${openaire_db_name}.software s  LATERAL VIEW explode(s.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
+
+CREATE TABLE ${stats_db_name}.software_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
+
+CREATE TABLE ${stats_db_name}.software_concepts AS SELECT substr(p.id, 4) AS id, contexts.context.id AS concept FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.context) contexts AS context;
+
+CREATE TABLE ${stats_db_name}.software_datasources AS SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource end as datasource FROM (SELECT  substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource 
+FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.instance) instances AS instance) p LEFT OUTER JOIN
+(SELECT substr(d.id, 4) id FROM ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d ON p.datasource = d.id;
+
+CREATE TABLE ${stats_db_name}.software_languages AS select substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.software p;
+
+CREATE TABLE ${stats_db_name}.software_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.originalid) oids AS ids;
+
+CREATE TABLE ${stats_db_name}.software_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.pid) pids AS ppid;
+
+CREATE TABLE ${stats_db_name}.software_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.software p LATERAL VIEW explode(p.subject) subjects AS subject;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_1.sql

@@ -1,9 +0,0 @@
-------------------------------------------------------
-------------------------------------------------------
--- 4. Dataset table/view and Dataset related tables/views
-------------------------------------------------------
-------------------------------------------------------
-
--- Dataset temporary table supporting updates
-DROP TABLE IF EXISTS ${stats_db_name}.dataset_tmp;
-CREATE TABLE ${stats_db_name}.dataset_tmp (id STRING, title STRING, publisher STRING, journal STRING, date STRING, year STRING, bestlicence STRING, embargo_end_date STRING, delayed BOOLEAN,   authors INT,   source STRING,   abstract BOOLEAN,   type STRING ) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_10.sql

@@ -1 +0,0 @@
-create table ${stats_db_name}.dataset_topics as select substr(p.id, 4) as id, subjects.subject.qualifier.classname as type, subjects.subject.value as topic from ${openaire_db_name}.dataset p lateral view explode(p.subject) subjects as subject;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_2.sql

@@ -1,7 +0,0 @@
-INSERT INTO ${stats_db_name}.dataset_tmp select substr(d.id, 4) as id, d.title[0].value as title, d.publisher.value as publisher, cast(null as string) as journal, 
-d.dateofacceptance.value as date, date_format(d.dateofacceptance.value,'yyyy') as year, d.bestaccessright.classname as bestlicence,
-d.embargoenddate.value as embargo_end_date, false as delayed, size(d.author) as authors , concat_ws('\u003B',d.source.value) as source,
- case when size(d.description) > 0 then true else false end as abstract,
-'dataset' as type
-from ${openaire_db_name}.dataset d
-where d.datainfo.deletedbyinference=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_3.sql

@@ -1,2 +0,0 @@
--- Dataset_citations
-Create table ${stats_db_name}.dataset_citations as select substr(d.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") as result from ${openaire_db_name}.dataset d  lateral view explode(d.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_4.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.dataset_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_5.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.dataset_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.context) contexts as context;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_6.sql

@@ -1,3 +0,0 @@
-CREATE TABLE ${stats_db_name}.dataset_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT  substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource 
-from ${openaire_db_name}.dataset p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
-(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_7.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.dataset_languages AS select substr(p.id, 4) as id, p.language.classname as language from ${openaire_db_name}.dataset p;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_8.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.dataset_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.dataset p LATERAL VIEW explode(p.originalid) oids as ids;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step4_9.sql

@@ -1 +0,0 @@
-create table ${stats_db_name}.dataset_pids as select substr(p.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.dataset p lateral view explode(p.pid) pids as ppid;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5.sql

@@ -0,0 +1,37 @@
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+-- Otherresearchproduct table/view and Otherresearchproduct related tables/views
+--------------------------------------------------------------------------------
+--------------------------------------------------------------------------------
+
+-- Otherresearchproduct temporary table supporting updates
+DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_tmp;
+CREATE TABLE ${stats_db_name}.otherresearchproduct_tmp (   id STRING,   title STRING,   publisher STRING,   journal STRING,   date STRING,   year STRING,   bestlicence STRING,   embargo_end_date STRING,   delayed BOOLEAN,   authors INT,   source STRING,   abstract BOOLEAN,   type STRING )  CLUSTERED BY (id) INTO 100 buckets stored AS orc tblproperties('transactional'='true');
+
+INSERT INTO ${stats_db_name}.otherresearchproduct_tmp SELECT substr(o.id, 4) AS id, o.title[0].value AS title, o.publisher.value AS publisher, CAST(NULL AS string) AS journal, 
+o.dateofacceptance.value AS DATE, date_format(o.dateofacceptance.value,'yyyy') AS year, o.bestaccessright.classname AS bestlicence,
+o.embargoenddate.value as embargo_end_date, FALSE AS delayed, SIZE(o.author) AS authors , concat_ws('\u003B',o.source.value) AS source,
+CASE WHEN SIZE(o.description) > 0 THEN TRUE ELSE FALSE END AS abstract,
+'other' AS type 
+FROM ${openaire_db_name}.otherresearchproduct o
+WHERE o.datainfo.deletedbyinference=FALSE;
+
+-- Otherresearchproduct_citations
+CREATE TABLE ${stats_db_name}.otherresearchproduct_citations AS SELECT substr(o.id, 4) AS id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") AS RESULT FROM ${openaire_db_name}.otherresearchproduct o  LATERAL VIEW explode(o.extrainfo) citations AS citation WHERE xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";
+
+CREATE TABLE ${stats_db_name}.otherresearchproduct_classifications AS SELECT substr(p.id, 4) AS id, instancetype.classname AS type FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances AS instancetype;
+
+CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts AS SELECT substr(p.id, 4) AS id, contexts.context.id AS concept FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts AS context;
+
+
+CREATE TABLE ${stats_db_name}.otherresearchproduct_datasources AS SELECT p.id, CASE WHEN d.id IS NULL THEN 'other' ELSE p.datasource END AS datasource FROM (SELECT  substr(p.id, 4) AS id, substr(instances.instance.hostedby.key, 4) AS datasource 
+from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
+(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;
+
+CREATE TABLE ${stats_db_name}.otherresearchproduct_languages AS SELECT substr(p.id, 4) AS id, p.language.classname AS language FROM ${openaire_db_name}.otherresearchproduct p;
+
+CREATE TABLE ${stats_db_name}.otherresearchproduct_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids AS ids;
+
+CREATE TABLE ${stats_db_name}.otherresearchproduct_pids AS SELECT substr(p.id, 4) AS id, ppid.qualifier.classname AS type, ppid.value AS pid FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.pid) pids AS ppid;
+
+CREATE TABLE ${stats_db_name}.otherresearchproduct_topics AS SELECT substr(p.id, 4) AS id, subjects.subject.qualifier.classname AS type, subjects.subject.value AS topic FROM ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.subject) subjects AS subject;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_1.sql

@@ -1,9 +0,0 @@
---------------------------------------------------------
---------------------------------------------------------
--- 5. Software table/view and Software related tables/views
---------------------------------------------------------
---------------------------------------------------------
-
--- Software temporary table supporting updates
-DROP TABLE IF EXISTS ${stats_db_name}.software_tmp;
-CREATE TABLE ${stats_db_name}.software_tmp (id STRING,   title STRING,   publisher STRING,   journal STRING,   date STRING,   year STRING,   bestlicence STRING,   embargo_end_date STRING,   delayed BOOLEAN,   authors INT,   source STRING,   abstract BOOLEAN,   type STRING )  clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_10.sql

@@ -1 +0,0 @@
-create table ${stats_db_name}.software_topics as select substr(p.id, 4) as id, subjects.subject.qualifier.classname as type, subjects.subject.value as topic from ${openaire_db_name}.software p lateral view explode(p.subject) subjects as subject;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_2.sql

@@ -1,7 +0,0 @@
-INSERT INTO ${stats_db_name}.software_tmp select substr(s.id, 4) as id, s.title[0].value as title, s.publisher.value as publisher, cast(null as string) as journal, 
-s.dateofacceptance.value as date, date_format(s.dateofacceptance.value,'yyyy') as year, s.bestaccessright.classname as bestlicence,
-s.embargoenddate.value as embargo_end_date, false as delayed, size(s.author) as authors , concat_ws('\u003B',s.source.value) as source,
- case when size(s.description) > 0 then true else false end as abstract,
-'software' as type
-from ${openaire_db_name}.software s
-where s.datainfo.deletedbyinference=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_3.sql

@@ -1,2 +0,0 @@
--- Software_citations
-Create table ${stats_db_name}.software_citations as select substr(s.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") as result from ${openaire_db_name}.software s  lateral view explode(s.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_4.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.software_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.software p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_5.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.software_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.software p LATERAL VIEW explode(p.context) contexts as context;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_6.sql

@@ -1,3 +0,0 @@
-CREATE TABLE ${stats_db_name}.software_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT  substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource 
-from ${openaire_db_name}.software p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
-(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_7.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.software_languages AS select substr(p.id, 4) as id, p.language.classname as language from ${openaire_db_name}.software p;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_8.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.software_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.software p LATERAL VIEW explode(p.originalid) oids as ids;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step5_9.sql

@@ -1 +0,0 @@
-create table ${stats_db_name}.software_pids as select substr(p.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.software p lateral view explode(p.pid) pids as ppid;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6.sql

@@ -0,0 +1,30 @@
+-- noinspection SqlNoDataSourceInspectionForFile
+
+------------------------------------------------------
+------------------------------------------------------
+-- Project table/view and Project related tables/views
+------------------------------------------------------
+------------------------------------------------------
+-- Project_oids Table
+DROP TABLE IF EXISTS ${stats_db_name}.project_oids;
+CREATE TABLE ${stats_db_name}.project_oids AS SELECT substr(p.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids AS ids;
+
+-- Project_organizations Table
+DROP TABLE IF EXISTS ${stats_db_name}.project_organizations;
+CREATE TABLE ${stats_db_name}.project_organizations AS SELECT substr(r.source, 4) AS id, substr(r.target, 4) AS organization from ${openaire_db_name}.relation r WHERE r.reltype='projectOrganization';
+
+-- Project_results Table
+DROP TABLE IF EXISTS ${stats_db_name}.project_results;
+CREATE TABLE ${stats_db_name}.project_results AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS result FROM ${openaire_db_name}.relation r WHERE r.reltype='resultProject' and r.datainfo.deletedbyinference=false;
+
+-- Project table
+----------------
+-- Creating and populating temporary Project table
+DROP TABLE IF EXISTS ${stats_db_name}.project_tmp;
+CREATE TABLE ${stats_db_name}.project_tmp (id STRING, acronym STRING, title STRING, funder STRING, funding_lvl0 STRING, funding_lvl1 STRING, funding_lvl2 STRING, ec39 STRING, type STRING, startdate STRING, enddate STRING, start_year INT, end_year INT, duration INT, haspubs STRING, numpubs INT, daysforlastpub INT, delayedpubs INT, callidentifier STRING, code STRING) CLUSTERED BY (id) INTO 100 buckets stored AS orc tblproperties('transactional'='true');
+
+INSERT INTO ${stats_db_name}.project_tmp SELECT substr(p.id, 4) AS id, p.acronym.value AS acronym, p.title.value AS title, xpath_string(p.fundingtree[0].value, '//funder/name') AS funder, xpath_string(p.fundingtree[0].value, '//funding_level_0/name') AS funding_lvl0, xpath_string(p.fundingtree[0].value, '//funding_level_1/name') AS funding_lvl1, xpath_string(p.fundingtree[0].value, '//funding_level_2/name') AS funding_lvl2, p.ecsc39.value AS ec39, p.contracttype.classname AS type, p.startdate.value AS startdate, p.enddate.value AS enddate,  year(p.startdate.value) AS start_year, year(p.enddate.value) AS end_year, CAST(MONTHS_BETWEEN(p.enddate.value, p.startdate.value) AS INT) AS duration, 'no' AS haspubs, 0 AS numpubs, 0 AS daysforlastpub, 0 AS delayedpubs, p.callidentifier.value AS callidentifier, p.code.value AS code FROM ${openaire_db_name}.project p WHERE p.datainfo.deletedbyinference=false;
+
+create table ${stats_db_name}.funder as
+select distinct  xpath_string(fund, '//funder/id') as id, xpath_string(fund, '//funder/name') as name, xpath_string(fund, '//funder/shortname') as shortname
+from ${openaire_db_name}.project p lateral view explode(p.fundingtree.value) fundingtree as fund

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_1.sql

@@ -1,9 +0,0 @@
---------------------------------------------------------------------------------
---------------------------------------------------------------------------------
--- 6. Otherresearchproduct table/view and Otherresearchproduct related tables/views
---------------------------------------------------------------------------------
---------------------------------------------------------------------------------
-
--- Otherresearchproduct temporary table supporting updates
-DROP TABLE IF EXISTS ${stats_db_name}.otherresearchproduct_tmp;
-CREATE TABLE ${stats_db_name}.otherresearchproduct_tmp (   id STRING,   title STRING,   publisher STRING,   journal STRING,   date STRING,   year STRING,   bestlicence STRING,   embargo_end_date STRING,   delayed BOOLEAN,   authors INT,   source STRING,   abstract BOOLEAN,   type STRING )  clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_10.sql

@@ -1 +0,0 @@
-create table ${stats_db_name}.otherresearchproduct_topics as select substr(p.id, 4) as id, subjects.subject.qualifier.classname as type, subjects.subject.value as topic from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.subject) subjects as subject;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_2.sql

@@ -1,7 +0,0 @@
-INSERT INTO ${stats_db_name}.otherresearchproduct_tmp select substr(o.id, 4) as id, o.title[0].value as title, o.publisher.value as publisher, cast(null as string) as journal, 
-o.dateofacceptance.value as date, date_format(o.dateofacceptance.value,'yyyy') as year, o.bestaccessright.classname as bestlicence,
-o.embargoenddate.value as embargo_end_date, false as delayed, size(o.author) as authors , concat_ws('\u003B',o.source.value) as source,
-case when size(o.description) > 0 then true else false end as abstract,
-'other' as type 
-from ${openaire_db_name}.otherresearchproduct o
-where o.datainfo.deletedbyinference=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_3.sql

@@ -1,2 +0,0 @@
--- Otherresearchproduct_citations
-Create table ${stats_db_name}.otherresearchproduct_citations as select substr(o.id, 4) as id, xpath_string(citation.value, "//citation/id[@type='openaire']/@value") as result from ${openaire_db_name}.otherresearchproduct o  lateral view explode(o.extrainfo) citations as citation where xpath_string(citation.value, "//citation/id[@type='openaire']/@value") !="";

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_4.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.otherresearchproduct_classifications AS SELECT substr(p.id, 4) as id, instancetype.classname as type from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.instance.instancetype) instances as instancetype;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_5.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.otherresearchproduct_concepts AS SELECT substr(p.id, 4) as id, contexts.context.id as concept from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.context) contexts as context;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_6.sql

@@ -1,3 +0,0 @@
-CREATE TABLE ${stats_db_name}.otherresearchproduct_datasources as SELECT p.id, case when d.id is null then 'other' else p.datasource end as datasource FROM (SELECT  substr(p.id, 4) as id, substr(instances.instance.hostedby.key, 4) as datasource 
-from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.instance) instances as instance) p LEFT OUTER JOIN
-(SELECT substr(d.id, 4) id from ${openaire_db_name}.datasource d WHERE d.datainfo.deletedbyinference=false) d on p.datasource = d.id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_7.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.otherresearchproduct_languages AS select substr(p.id, 4) as id, p.language.classname as language from ${openaire_db_name}.otherresearchproduct p;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_8.sql

@@ -1 +0,0 @@
-CREATE TABLE ${stats_db_name}.otherresearchproduct_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.otherresearchproduct p LATERAL VIEW explode(p.originalid) oids as ids;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step6_9.sql

@@ -1 +0,0 @@
-create table ${stats_db_name}.otherresearchproduct_pids as select substr(p.id, 4) as id, ppid.qualifier.classname as type, ppid.value as pid from ${openaire_db_name}.otherresearchproduct p lateral view explode(p.pid) pids as ppid;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7.sql

@@ -0,0 +1,31 @@
+----------------------------------------------------
+----------------------------------------------------
+-- Result table/view and Result related tables/views
+----------------------------------------------------
+----------------------------------------------------
+
+-- Views on temporary tables that should be re-created in the end
+CREATE OR REPLACE VIEW ${stats_db_name}.result as SELECT *, bestlicence AS access_mode FROM ${stats_db_name}.publication_tmp UNION ALL SELECT *,bestlicence AS access_mode FROM ${stats_db_name}.software_tmp UNION ALL SELECT *,bestlicence AS access_mode FROM ${stats_db_name}.dataset_tmp UNION ALL SELECT *,bestlicence AS access_mode FROM ${stats_db_name}.otherresearchproduct_tmp;
+
+-- Views on final tables
+CREATE OR REPLACE VIEW ${stats_db_name}.result_datasources AS SELECT * FROM ${stats_db_name}.publication_datasources UNION ALL SELECT * FROM ${stats_db_name}.software_datasources UNION ALL SELECT * FROM ${stats_db_name}.dataset_datasources UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_datasources;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.result_citations AS SELECT * FROM ${stats_db_name}.publication_citations UNION ALL SELECT * FROM ${stats_db_name}.software_citations UNION ALL SELECT * FROM ${stats_db_name}.dataset_citations UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_citations;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.result_classifications AS SELECT * FROM ${stats_db_name}.publication_classifications UNION ALL SELECT * FROM ${stats_db_name}.software_classifications UNION ALL SELECT * FROM ${stats_db_name}.dataset_classifications UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_classifications;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.result_concepts AS SELECT * FROM ${stats_db_name}.publication_concepts UNION ALL SELECT * FROM ${stats_db_name}.software_concepts UNION ALL SELECT * FROM ${stats_db_name}.dataset_concepts UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_concepts;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.result_languages AS SELECT * FROM ${stats_db_name}.publication_languages UNION ALL SELECT * FROM ${stats_db_name}.software_languages UNION ALL SELECT * FROM ${stats_db_name}.dataset_languages UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_languages;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.result_oids AS SELECT * FROM ${stats_db_name}.publication_oids UNION ALL SELECT * FROM ${stats_db_name}.software_oids UNION ALL SELECT * FROM ${stats_db_name}.dataset_oids UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_oids;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.result_pids AS SELECT * FROM ${stats_db_name}.publication_pids UNION ALL SELECT * FROM ${stats_db_name}.software_pids UNION ALL SELECT * FROM ${stats_db_name}.dataset_pids UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_pids;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.result_topics AS SELECT * FROM ${stats_db_name}.publication_topics UNION ALL SELECT * FROM ${stats_db_name}.software_topics UNION ALL SELECT * FROM ${stats_db_name}.dataset_topics UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_topics;
+
+DROP TABLE IF EXISTS ${stats_db_name}.result_organization;
+CREATE TABLE ${stats_db_name}.result_organization AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization FROM ${openaire_db_name}.relation r WHERE r.reltype='resultOrganization';
+
+DROP TABLE IF EXISTS ${stats_db_name}.result_projects;
+CREATE TABLE ${stats_db_name}.result_projects AS select pr.result AS id, pr.id AS project, datediff(p.enddate, p.startdate) AS daysfromend FROM ${stats_db_name}.result r JOIN ${stats_db_name}.project_results pr ON r.id=pr.result JOIN ${stats_db_name}.project_tmp p ON p.id=pr.id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7_1.sql

@@ -1,8 +0,0 @@
-------------------------------------------------------
-------------------------------------------------------
--- 7. Project table/view and Project related tables/views
-------------------------------------------------------
-------------------------------------------------------
--- Project_oids Table
-DROP TABLE IF EXISTS ${stats_db_name}.project_oids;
-CREATE TABLE ${stats_db_name}.project_oids AS SELECT substr(p.id, 4) as id, oids.ids as oid from ${openaire_db_name}.project p LATERAL VIEW explode(p.originalid) oids as ids;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7_2.sql

@@ -1,3 +0,0 @@
--- Project_organizations Table
-DROP TABLE IF EXISTS ${stats_db_name}.project_organizations;
-CREATE TABLE ${stats_db_name}.project_organizations AS SELECT substr(r.source, 4) as id, substr(r.target, 4) AS organization from ${openaire_db_name}.relation r WHERE r.reltype='projectOrganization';

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7_3.sql

@@ -1,3 +0,0 @@
--- Project_results Table
-DROP TABLE IF EXISTS ${stats_db_name}.project_results;
-CREATE TABLE ${stats_db_name}.project_results AS SELECT substr(r.target, 4) as id, substr(r.source, 4) AS result from ${openaire_db_name}.relation r WHERE r.reltype='resultProject';

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7_4.sql

@@ -1,5 +0,0 @@
--- Project table
-----------------
--- Creating and populating temporary Project table
-DROP TABLE IF EXISTS ${stats_db_name}.project_tmp;
-CREATE TABLE ${stats_db_name}.project_tmp (id STRING, acronym STRING, title STRING, funder STRING, funding_lvl0 STRING, funding_lvl1 STRING, funding_lvl2 STRING, ec39 STRING, type STRING, startdate STRING, enddate STRING, start_year STRING, end_year STRING, duration INT, haspubs STRING, numpubs INT, daysforlastpub INT, delayedpubs INT, callidentifier STRING, code STRING) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step7_5.sql

@@ -1 +0,0 @@
-INSERT INTO ${stats_db_name}.project_tmp SELECT substr(p.id, 4) as id, p.acronym.value as acronym, p.title.value as title, xpath_string(p.fundingtree[0].value, '//funder/name') as funder, xpath_string(p.fundingtree[0].value, '//funding_level_0/name') as funding_lvl0, xpath_string(p.fundingtree[0].value, '//funding_level_1/name') as funding_lvl1, xpath_string(p.fundingtree[0].value, '//funding_level_2/name') as funding_lvl2, p.ecsc39.value as ec39, p.contracttype.classname as type, p.startdate.value as startdate, p.enddate.value as enddate,  date_format(p.startdate.value, 'yyyy') as start_year, date_format(p.enddate.value, 'yyyy') as end_year, 0 as duration, 'no' as haspubs, 0 as numpubs, 0 as daysforlastpub, 0 as delayedpubs, p.callidentifier.value as callidentifier, p.code.value as code from ${openaire_db_name}.project p WHERE p.datainfo.deletedbyinference=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8.sql

@@ -0,0 +1,58 @@
+-- noinspection SqlNoDataSourceInspectionForFile
+
+------------------------------------------------------------
+------------------------------------------------------------
+-- Datasource table/view and Datasource related tables/views
+------------------------------------------------------------
+------------------------------------------------------------
+
+-- Datasource table creation & update
+-------------------------------------
+-- Creating and populating temporary datasource table
+DROP TABLE IF EXISTS ${stats_db_name}.datasource_tmp;
+CREATE TABLE ${stats_db_name}.datasource_tmp(`id` string, `name` STRING, `type` STRING, `dateofvalidation` STRING, `yearofvalidation` string, `harvested` BOOLEAN, `piwik_id` INT, `latitude` STRING, `longitude`STRING, `websiteurl` STRING, `compatibility` STRING) CLUSTERED BY (id) INTO 100 buckets stored AS orc tblproperties('transactional'='true');
+
+-- Insert statement that takes into account the piwik_id of the openAIRE graph
+INSERT INTO ${stats_db_name}.datasource_tmp 
+SELECT substr(d1.id, 4) AS id, officialname.value AS name, 
+datasourcetype.classname AS type, dateofvalidation.value AS dateofvalidation, date_format(d1.dateofvalidation.value,'yyyy') AS yearofvalidation, 
+FALSE AS harvested, 
+CASE WHEN d2.piwik_id IS NULL THEN 0 ELSE d2.piwik_id END AS piwik_id, 
+d1.latitude.value AS latitude, d1.longitude.value AS longitude, 
+d1.websiteurl.value AS websiteurl, d1.openairecompatibility.classid AS compatibility
+FROM ${openaire_db_name}.datasource d1
+LEFT OUTER JOIN
+(SELECT id, split(originalidd, '\\:')[1] as piwik_id
+FROM ${openaire_db_name}.datasource
+LATERAL VIEW EXPLODE(originalid) temp AS originalidd
+WHERE originalidd like "piwik:%") AS d2
+ON d1.id = d2.id
+WHERE d1.datainfo.deletedbyinference=FALSE;
+
+-- Updating temporary table with everything that is not based on results -> This is done with the following "dual" table.
+-- Creating a temporary dual table that will be removed after the following insert
+CREATE TABLE ${stats_db_name}.dual(dummy CHAR(1));
+INSERT INTO ${stats_db_name}.dual VALUES('X');
+INSERT INTO ${stats_db_name}.datasource_tmp (`id`, `name`, `type`, `dateofvalidation`, `yearofvalidation`, `harvested`, `piwik_id`, `latitude`, `longitude`, `websiteurl`, `compatibility`) 
+SELECT 'other', 'Other', 'Repository', NULL, NULL, false, 0, NULL, NULL, NULL, 'unknown' FROM ${stats_db_name}.dual WHERE 'other' not in (SELECT id FROM ${stats_db_name}.datasource_tmp WHERE name='Unknown Repository');
+DROP TABLE ${stats_db_name}.dual;
+
+UPDATE ${stats_db_name}.datasource_tmp SET name='Other' WHERE name='Unknown Repository';
+UPDATE ${stats_db_name}.datasource_tmp SET yearofvalidation=null WHERE yearofvalidation='-1';
+
+DROP TABLE IF EXISTS ${stats_db_name}.datasource_languages;
+CREATE TABLE ${stats_db_name}.datasource_languages AS SELECT substr(d.id, 4) AS id, langs.languages AS language FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.odlanguages.value) langs AS languages;
+
+DROP TABLE IF EXISTS ${stats_db_name}.datasource_oids;
+CREATE TABLE ${stats_db_name}.datasource_oids AS SELECT substr(d.id, 4) AS id, oids.ids AS oid FROM ${openaire_db_name}.datasource d LATERAL VIEW explode(d.originalid) oids AS ids;
+
+DROP TABLE IF EXISTS ${stats_db_name}.datasource_organizations;
+CREATE TABLE ${stats_db_name}.datasource_organizations AS SELECT substr(r.target, 4) AS id, substr(r.source, 4) AS organization FROM ${openaire_db_name}.relation r WHERE r.reltype='datasourceOrganization';
+
+-- datasource sources:
+-- where the datasource info have been collected from.
+create table if not exists ${stats_db_name}.datasource_sources AS select substr(d.id,4) as id, substr(cf.key, 4) as datasource from ${openaire_db_name}.datasource d lateral view explode(d.collectedfrom) cfrom as cf where d.datainfo.deletedbyinference=false;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.datasource_results AS SELECT datasource AS id, id AS result FROM ${stats_db_name}.result_datasources;
+
+

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_1.sql

@@ -1,8 +0,0 @@
-----------------------------------------------------
-----------------------------------------------------
--- 8. Result table/view and Result related tables/views
-----------------------------------------------------
-----------------------------------------------------
-
--- Views on temporary tables that should be re-created in the end
-CREATE OR REPLACE VIEW ${stats_db_name}.result as SELECT *, bestlicence as access_mode FROM ${stats_db_name}.publication_tmp UNION ALL SELECT *,bestlicence as access_mode FROM ${stats_db_name}.software_tmp UNION ALL SELECT *,bestlicence as access_mode FROM ${stats_db_name}.dataset_tmp UNION ALL SELECT *,bestlicence as access_mode FROM ${stats_db_name}.otherresearchproduct_tmp;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_10.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS ${stats_db_name}.result_organization;
-CREATE TABLE ${stats_db_name}.result_organization AS SELECT substr(r.target, 4) as id, substr(r.source, 4) as organization from ${openaire_db_name}.relation r where r.reltype='resultOrganization';

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_11.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS ${stats_db_name}.result_projects;
-CREATE TABLE ${stats_db_name}.result_projects AS select pr.result as id, pr.id as project, datediff(p.enddate, p.startdate) as daysfromend from ${stats_db_name}.result r join ${stats_db_name}.project_results pr on r.id=pr.result join ${stats_db_name}.project_tmp p on p.id=pr.id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_2.sql

@@ -1,2 +0,0 @@
--- Views on final tables
-CREATE OR REPLACE VIEW ${stats_db_name}.result_datasources as SELECT * FROM ${stats_db_name}.publication_datasources UNION ALL SELECT * FROM ${stats_db_name}.software_datasources UNION ALL SELECT * FROM ${stats_db_name}.dataset_datasources UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_datasources;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_3.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.result_citations as SELECT * FROM ${stats_db_name}.publication_citations UNION ALL SELECT * FROM ${stats_db_name}.software_citations UNION ALL SELECT * FROM ${stats_db_name}.dataset_citations UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_citations;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_4.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.result_classifications as SELECT * FROM ${stats_db_name}.publication_classifications UNION ALL SELECT * FROM ${stats_db_name}.software_classifications UNION ALL SELECT * FROM ${stats_db_name}.dataset_classifications UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_classifications;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_5.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.result_concepts as SELECT * FROM ${stats_db_name}.publication_concepts UNION ALL SELECT * FROM ${stats_db_name}.software_concepts UNION ALL SELECT * FROM ${stats_db_name}.dataset_concepts UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_concepts;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_6.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.result_languages as SELECT * FROM ${stats_db_name}.publication_languages UNION ALL SELECT * FROM ${stats_db_name}.software_languages UNION ALL SELECT * FROM ${stats_db_name}.dataset_languages UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_languages;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_7.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.result_oids as SELECT * FROM ${stats_db_name}.publication_oids UNION ALL SELECT * FROM ${stats_db_name}.software_oids UNION ALL SELECT * FROM ${stats_db_name}.dataset_oids UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_oids;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_8.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.result_pids as SELECT * FROM ${stats_db_name}.publication_pids UNION ALL SELECT * FROM ${stats_db_name}.software_pids UNION ALL SELECT * FROM ${stats_db_name}.dataset_pids UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_pids;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step8_9.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.result_topics as SELECT * FROM ${stats_db_name}.publication_topics UNION ALL SELECT * FROM ${stats_db_name}.software_topics UNION ALL SELECT * FROM ${stats_db_name}.dataset_topics UNION ALL SELECT * FROM ${stats_db_name}.otherresearchproduct_topics;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9.sql

@@ -0,0 +1,12 @@
+----------------------------------------------------------------
+----------------------------------------------------------------
+-- Organization table/view and Organization related tables/views
+----------------------------------------------------------------
+----------------------------------------------------------------
+DROP TABLE IF EXISTS ${stats_db_name}.organization;
+CREATE TABLE IF NOT EXISTS ${stats_db_name}.organization AS SELECT substr(o.id, 4) as id, o.legalname.value as name, o.legalshortname.value as legalshortname, o.country.classid as country
+FROM ${openaire_db_name}.organization o WHERE o.datainfo.deletedbyinference=FALSE;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.organization_datasources AS SELECT organization AS id, id AS datasource FROM ${stats_db_name}.datasource_organizations;
+
+CREATE OR REPLACE VIEW ${stats_db_name}.organization_projects AS SELECT id AS project, organization as id FROM ${stats_db_name}.project_organizations;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_1.sql

@@ -1,10 +0,0 @@
-------------------------------------------------------------
-------------------------------------------------------------
--- 9. Datasource table/view and Datasource related tables/views
-------------------------------------------------------------
-------------------------------------------------------------
--- Datasource table creation & update
--------------------------------------
--- Creating and populating temporary datasource table
-DROP TABLE IF EXISTS ${stats_db_name}.datasource_tmp;
-create table ${stats_db_name}.datasource_tmp(`id` string, `name` string, `type` string, `dateofvalidation` string, `yearofvalidation` string, `harvested` boolean, `piwik_id` int, `latitude` string, `longitude` string, `websiteurl` string, `compatibility` string) clustered by (id) into 100 buckets stored as orc tblproperties('transactional'='true');

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_2.sql

@@ -1,3 +0,0 @@
-insert into ${stats_db_name}.datasource_tmp SELECT substr(d.id, 4) as id, officialname.value as name, datasourcetype.classname as type, dateofvalidation.value as dateofvalidation, date_format(d.dateofvalidation.value,'yyyy') as yearofvalidation, false as harvested, 0 as piwik_id, d.latitude.value as latitude, d.longitude.value as longitude, d.websiteurl.value as websiteurl, d.openairecompatibility.classid as compatibility
-from ${openaire_db_name}.datasource d
-WHERE d.datainfo.deletedbyinference=false;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_3.sql

@@ -1,7 +0,0 @@
--- Updating temporary table with everything that is not based on results -> This is done with the following "dual" table. To see if default values are there
--- Creating a temporary dual table that will be removed after the following insert
-CREATE TABLE ${stats_db_name}.dual(dummy char(1));
-INSERT INTO ${stats_db_name}.dual values('X');
-INSERT INTO ${stats_db_name}.datasource_tmp (`id`, `name`, `type`, `dateofvalidation`, `yearofvalidation`, `harvested`, `piwik_id`, `latitude`, `longitude`, `websiteurl`, `compatibility`) 
-SELECT 'other', 'Other', 'Repository', NULL, NULL, false, 0, NULL, NULL, NULL, 'unknown' FROM ${stats_db_name}.dual WHERE 'other' not in (SELECT id FROM ${stats_db_name}.datasource_tmp WHERE name='Unknown Repository');
-DROP TABLE ${stats_db_name}.dual;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_4.sql

@@ -1 +0,0 @@
-UPDATE ${stats_db_name}.datasource_tmp SET name='Other' where name='Unknown Repository';

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_5.sql

@@ -1 +0,0 @@
-UPDATE ${stats_db_name}.datasource_tmp SET yearofvalidation=null WHERE yearofvalidation='-1';

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_6.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS ${stats_db_name}.datasource_languages;
-CREATE TABLE ${stats_db_name}.datasource_languages AS SELECT substr(d.id, 4) as id, langs.languages as language from ${openaire_db_name}.datasource d LATERAL VIEW explode(d.odlanguages.value) langs as languages;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_7.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS ${stats_db_name}.datasource_oids;
-CREATE TABLE ${stats_db_name}.datasource_oids AS SELECT substr(d.id, 4) as id, oids.ids as oid from ${openaire_db_name}.datasource d LATERAL VIEW explode(d.originalid) oids as ids;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_8.sql

@@ -1,2 +0,0 @@
-DROP TABLE IF EXISTS ${stats_db_name}.datasource_organizations;
-CREATE TABLE ${stats_db_name}.datasource_organizations AS select substr(r.target, 4) as id, substr(r.source, 4) as organization from ${openaire_db_name}.relation r where r.reltype='datasourceOrganization';

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step9_9.sql

@@ -1 +0,0 @@
-CREATE OR REPLACE VIEW ${stats_db_name}.datasource_results AS SELECT datasource AS id, id AS result FROM ${stats_db_name}.result_datasources;