From 73656f7f3171a7937543e399e118575ca04e622a Mon Sep 17 00:00:00 2001 From: Spyros Zoupanos Date: Sat, 26 Sep 2020 11:18:03 +0300 Subject: [PATCH] More corrections on the portalStats queries --- .../graph/usagestats/export/PiwikStatsDB.java | 36 ++++++++++++------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java index 7782a14796..50ea4c3f41 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java @@ -812,8 +812,24 @@ public class PiwikStatsDB { Statement stmt = con.createStatement(); con.setAutoCommit(false); +// Original queries where of the style +// +// SELECT DISTINCT source, id_visit, country, action, url, roid.oid, 'oaItem', `timestamp`, referrer_name, agent +// FROM usagestats_20200907.process_portal_log_tmp2, +// openaire_prod_stats_20200821.result_oids roid +// WHERE entity_id IS NOT null AND entity_id=roid.oid AND roid.oid IS NOT null +// // The following query is an example of how queries should be // +// +// INSERT INTO usagestats_20200907.piwiklogtmp +// SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'oaItem', `timestamp`, referrer_name, agent +// FROM usagestats_20200907.process_portal_log_tmp +// WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id +// IN (SELECT roid.oid FROM openaire_prod_stats_20200821.result_oids roid WHERE roid.oid IS NOT NULL); +// +// We should consider if we would like the queries to be as the following +// // INSERT INTO usagestats_20200907.piwiklogtmp // SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'oaItem', `timestamp`, referrer_name, agent // FROM usagestats_20200907.process_portal_log_tmp @@ -826,10 +842,9 @@ public class PiwikStatsDB { "SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'oaItem', `timestamp`, referrer_name, agent " + "FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp " + - "WHERE process_portal_log_tmp.entity_id IS NOT NULL AND " + - "process_portal_log_tmp.entity_id != '' AND process_portal_log_tmp.entity_id " + + "WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id " + "IN (SELECT roid.oid FROM " + ConnectDB.getStatsDBSchema() - + ".result_oids roid roid WHERE roid.oid IS NOT NULL AND roid.oid != '')"; + + ".project_oids roid WHERE roid.oid IS NOT NULL)"; stmt.executeUpdate(sql); stmt.close(); @@ -839,10 +854,9 @@ public class PiwikStatsDB { "SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'datasource', `timestamp`, referrer_name, agent " + "FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp " + - "WHERE process_portal_log_tmp.entity_id IS NOT NULL AND " + - "process_portal_log_tmp.entity_id != '' AND process_portal_log_tmp.entity_id " + + "WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id " + "IN (SELECT roid.oid FROM " + ConnectDB.getStatsDBSchema() - + ".datasource_oids roid WHERE roid.oid IS NOT NULL AND roid.oid != '')"; + + ".project_oids roid WHERE roid.oid IS NOT NULL)"; stmt.executeUpdate(sql); stmt.close(); @@ -852,10 +866,9 @@ public class PiwikStatsDB { "SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'organization', `timestamp`, referrer_name, agent " + "FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp " + - "WHERE process_portal_log_tmp.entity_id IS NOT NULL AND " + - "process_portal_log_tmp.entity_id != '' AND process_portal_log_tmp.entity_id " + + "WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id " + "IN (SELECT roid.oid FROM " + ConnectDB.getStatsDBSchema() - + ".organization_oids roid WHERE roid.oid IS NOT NULL AND roid.oid != '')"; + + ".project_oids roid WHERE roid.oid IS NOT NULL)"; // stmt.executeUpdate(sql); stmt.close(); @@ -865,10 +878,9 @@ public class PiwikStatsDB { "SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'project', `timestamp`, referrer_name, agent " + "FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp " + - "WHERE process_portal_log_tmp.entity_id IS NOT NULL AND " + - "process_portal_log_tmp.entity_id != '' AND process_portal_log_tmp.entity_id " + + "WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id " + "IN (SELECT roid.oid FROM " + ConnectDB.getStatsDBSchema() - + ".project_oids roid WHERE roid.oid IS NOT NULL AND roid.oid != '')"; + + ".project_oids roid WHERE roid.oid IS NOT NULL)"; stmt.executeUpdate(sql); stmt.close();