D-Net
/
dnet-hadoop
14
5
Fork 11
Code Issues 14 Pull Requests 13 Projects 2 Releases 2 Wiki Activity

added the final steps for the observatory promote wf and some cleanup

This commit is contained in:
Antonis Lempesis 2021-05-18 15:23:20 +03:00
parent 625d993cd9
commit 168edcbde3
2 changed files with 21 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
dhp-workflows/dhp-stats-promote/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/workflow.xml
View File

@ -16,6 +16,14 @@
<name>monitor_db_production_name</name> <name>monitor_db_production_name</name>
<description>the name of the monitor public database</description> <description>the name of the monitor public database</description>
</property> </property>
<property>
<name>observatory_db_name</name>
<description>the monitor database name</description>
</property>
<property>
<name>observatory_db_production_name</name>
<description>the name of the monitor public database</description>
</property>
<property> <property>
<name>stats_tool_api_url</name> <name>stats_tool_api_url</name>
<description>The url of the API of the stats tool. Is used to trigger the cache promote.</description> <description>The url of the API of the stats tool. Is used to trigger the cache promote.</description>
@ -77,6 +85,19 @@
<argument>${monitor_db_production_name}</argument> <argument>${monitor_db_production_name}</argument>
<file>updateProductionViews.sh</file> <file>updateProductionViews.sh</file>
</shell> </shell>
<ok to="updateObservatoryViews"/>
<error to="Kill"/>
</action>
<action name="updateObservatoryViews">
<shell xmlns="uri:oozie:shell-action:0.1">
<job-tracker>${jobTracker}</job-tracker>
<name-node>${nameNode}</name-node>
<exec>updateProductionViews.sh</exec>
<argument>${observatory_db_name}</argument>
<argument>${observatory_db_production_name}</argument>
<file>updateProductionViews.sh</file>
</shell>
<ok to="promoteCache"/> <ok to="promoteCache"/>
<error to="Kill"/> <error to="Kill"/>
</action> </action>

32
dhp-workflows/dhp-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/stats/oozie_app/scripts/step12.sql
View File

@ -45,35 +45,3 @@ FROM ${stats_db_name}.dataset
UNION ALL UNION ALL
SELECT *, bestlicence AS access_mode SELECT *, bestlicence AS access_mode
FROM ${stats_db_name}.otherresearchproduct; FROM ${stats_db_name}.otherresearchproduct;
-------------------------------------------------------------------------------
-- To see with Antonis if the following is needed and where it should be placed
-------------------------------------------------------------------------------
CREATE TABLE ${stats_db_name}.numbers_country AS
SELECT org.country AS country, count(distinct rd.datasource) AS datasources, count(distinct r.id) AS publications
FROM ${stats_db_name}.result r,
${stats_db_name}.result_datasources rd,
${stats_db_name}.datasource d,
${stats_db_name}.datasource_organizations dor,
${stats_db_name}.organization org
WHERE r.id = rd.id
AND rd.datasource = d.id
AND d.id = dor.id
AND dor.organization = org.id
AND r.type = 'publication'
AND r.bestlicence = 'Open Access'
GROUP BY org.country;
-- ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS;
-- ANALYZE TABLE ${stats_db_name}.datasource COMPUTE STATISTICS FOR COLUMNS;
-- ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS;
-- ANALYZE TABLE ${stats_db_name}.publication COMPUTE STATISTICS FOR COLUMNS;
-- ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS;
-- ANALYZE TABLE ${stats_db_name}.dataset COMPUTE STATISTICS FOR COLUMNS;
-- ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS;
-- ANALYZE TABLE ${stats_db_name}.software COMPUTE STATISTICS FOR COLUMNS;
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS;
-- ANALYZE TABLE ${stats_db_name}.otherresearchproduct COMPUTE STATISTICS FOR COLUMNS;
-- ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS;
-- ANALYZE TABLE ${stats_db_name}.numbers_country COMPUTE STATISTICS FOR COLUMNS;