using new indicators for the definition of peerreviewed, gold, and green

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16.sql

@@ -1,62 +0,0 @@
-----------------------------------------------------
--- Shortcuts for various definitions in stats db ---
-----------------------------------------------------
-
--- Peer reviewed:
--- Results that have been collected from Crossref
-create table ${stats_db_name}.result_peerreviewed as
-with peer_reviewed as (
-    select distinct r.id as id
-    from ${stats_db_name}.result r
-    join ${stats_db_name}.result_sources rs on rs.id=r.id
-    join ${stats_db_name}.datasource d on d.id=rs.datasource
-    where d.name='Crossref')
-select distinct peer_reviewed.id as id, true as peer_reviewed
-from peer_reviewed
-union all
-select distinct r.id as id, false as peer_reviewed
-from ${stats_db_name}.result r
-left outer join peer_reviewed pr on pr.id=r.id
-where pr.id is null;
-
--- Green OA:
--- OA results that are hosted by an Institutional repository and have NOT been harvested from a DOAJ journal.
-create table ${stats_db_name}.result_greenoa as
-with result_green as (
-    select distinct r.id as id
-    from ${stats_db_name}.result r
-    join ${stats_db_name}.result_datasources rd on rd.id=r.id
-    join ${stats_db_name}.datasource d on d.id=rd.datasource
-    left outer join (
-        select rd.id from ${stats_db_name}.result_datasources rd
-        join ${stats_db_name}.datasource d on rd.datasource=d.id
-        join ${stats_db_name}.datasource_sources sds on sds.id=d.id
-        join ${stats_db_name}.datasource sd on sd.id=sds.datasource
-        where sd.name='DOAJ-ARTICLES'
-    ) as doaj on doaj.id=r.id
-    where r.bestlicence in ('Open Access', 'Open Source') and d.type='Institutional Repository' and doaj.id is null)
-select distinct result_green.id, true as green
-from result_green
-union all
-select distinct r.id as id, false as green
-from ${stats_db_name}.result r
-left outer join result_green rg on rg.id=r.id
-where rg.id  is null;
-
--- GOLD OA:
--- OA results that have been harvested from a DOAJ journal.
-create table ${stats_db_name}.result_gold as
-with result_gold as (
-    select distinct r.id as id
-    from ${stats_db_name}.result r
-    join ${stats_db_name}.result_datasources rd on rd.id=r.id
-    join ${stats_db_name}.datasource d on d.id=rd.datasource
-    join ${stats_db_name}.datasource_sources sds on sds.id=d.id
-    join ${stats_db_name}.datasource sd on sd.id=sds.datasource
-    where r.type='publication' and r.bestlicence='Open Access' and sd.name='DOAJ-Articles')
-select distinct result_gold.id, true as gold
-from result_gold
-union all
-select distinct r.id, false as gold
-from ${stats_db_name}.result r
-where r.id not in (select id from result_gold);

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step16_1-definitions.sql

@@ -0,0 +1,22 @@
+----------------------------------------------------
+-- Shortcuts for various definitions in stats db ---
+----------------------------------------------------
+
+-- Peer reviewed:
+create table ${stats_db_name}.result_peerreviewed as
+select r.id as id, case when doi.doi_from_crossref=1 and grey.grey_lit=0 then true else false end as peer_reviewed
+from result r
+left outer join ${stats_db_name}.indi_pub_doi_from_crossref doi on doi.id=r.id
+left outer join ${stats_db_name}.indi_pub_grey_lit grey on grey.id=r.id;
+
+-- Green OA:
+create table ${stats_db_name}.result_greenoa as
+select r.id, case when green.green_oa=1 then true else false end as green
+from ${stats_db_name}.result r
+left outer join ${stats_db_name}.indi_pub_green_oa green on green.id=r.id;
+
+-- GOLD OA:
+create table ${stats_db_name}.result_gold as
+select r.id, case when green.green_oa=1 then true else false end as green
+from ${stats_db_name}.result r
+         left outer join ${stats_db_name}.indi_pub_gold_oa green on green.id=r.id;

dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml

@@ -239,14 +239,27 @@
 			<param>stats_db_name=${stats_db_name}</param>
 			<param>openaire_db_name=${openaire_db_name}</param>
         </hive2>
-        <ok to="Step16"/>
+        <ok to="Step16-createIndicatorsTables"/>
         <error to="Kill"/>
     </action>
 
-    <action name="Step16">
+    <action name="Step16-createIndicatorsTables">
+        <shell xmlns="uri:oozie:shell-action:0.1">
+            <job-tracker>${jobTracker}</job-tracker>
+            <name-node>${nameNode}</name-node>
+            <exec>indicators.sh</exec>
+            <argument>${stats_db_name}</argument>
+            <argument>${wf:appPath()}/scripts/step16-createIndicatorsTables.sql</argument>
+            <file>indicators.sh</file>
+        </shell>
+        <ok to="Step16_1-definitions"/>
+        <error to="Kill"/>
+    </action>
+
+    <action name="Step16_1-definitions">
         <hive2 xmlns="uri:oozie:hive2-action:0.1">
             <jdbc-url>${hive_jdbc_url}</jdbc-url>
-            <script>scripts/step16.sql</script>
+            <script>scripts/step16_1-definitions.sql</script>
             <param>stats_db_name=${stats_db_name}</param>
             <param>openaire_db_name=${openaire_db_name}</param>
         </hive2>
@@ -272,24 +285,11 @@
             <param>stats_db_name=${stats_db_name}</param>
             <param>openaire_db_name=${openaire_db_name}</param>
         </hive2>
-        <ok to="Step16_7-createIndicatorsTables"/>
+        <ok to="Step18-contexts"/>
         <error to="Kill"/>
     </action>
 
-    <action name="Step16_7-createIndicatorsTables">
+    <action name="Step18-contexts">
-        <shell xmlns="uri:oozie:shell-action:0.1">
-            <job-tracker>${jobTracker}</job-tracker>
-            <name-node>${nameNode}</name-node>
-            <exec>indicators.sh</exec>
-            <argument>${stats_db_name}</argument>
-            <argument>${wf:appPath()}/scripts/step16_7-createIndicatorsTables.sql</argument>
-            <file>indicators.sh</file>
-        </shell>
-        <ok to="Step17"/>
-        <error to="Kill"/>
-    </action>
-
-    <action name="Step17">
         <shell xmlns="uri:oozie:shell-action:0.1">
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>
@@ -298,11 +298,11 @@
             <argument>${stats_db_name}</argument>
             <file>contexts.sh</file>
         </shell>
-        <ok to="Step19"/>
+        <ok to="Step19-finalize"/>
         <error to="Kill"/>
     </action>
         
-    <action name="Step19">
+    <action name="Step19-finalize">
         <shell xmlns="uri:oozie:shell-action:0.1">
             <job-tracker>${jobTracker}</job-tracker>
             <name-node>${nameNode}</name-node>