spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

More corrections on the portalStats queries

This commit is contained in:
Spyros Zoupanos 2020-09-26 11:18:03 +03:00
parent a497b19b21
commit 73656f7f31
1 changed files with 24 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
View File

@ -812,8 +812,24 @@ public class PiwikStatsDB {
Statement stmt = con.createStatement();
con.setAutoCommit(false);
// Original queries where of the style
//
// SELECT DISTINCT source, id_visit, country, action, url, roid.oid, 'oaItem', `timestamp`, referrer_name, agent
// FROM usagestats_20200907.process_portal_log_tmp2,
// openaire_prod_stats_20200821.result_oids roid
// WHERE entity_id IS NOT null AND entity_id=roid.oid AND roid.oid IS NOT null
//
// The following query is an example of how queries should be
//
//
// INSERT INTO usagestats_20200907.piwiklogtmp
// SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'oaItem', `timestamp`, referrer_name, agent
// FROM usagestats_20200907.process_portal_log_tmp
// WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id
// IN (SELECT roid.oid FROM openaire_prod_stats_20200821.result_oids roid WHERE roid.oid IS NOT NULL);
//
// We should consider if we would like the queries to be as the following
//
// INSERT INTO usagestats_20200907.piwiklogtmp
// SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'oaItem', `timestamp`, referrer_name, agent
// FROM usagestats_20200907.process_portal_log_tmp
@ -826,10 +842,9 @@ public class PiwikStatsDB {
"SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'oaItem', `timestamp`, referrer_name, agent "
+
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp " +
"WHERE process_portal_log_tmp.entity_id IS NOT NULL AND " +
"process_portal_log_tmp.entity_id != '' AND process_portal_log_tmp.entity_id " +
"WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id " +
"IN (SELECT roid.oid FROM " + ConnectDB.getStatsDBSchema()
+ ".result_oids roid roid WHERE roid.oid IS NOT NULL AND roid.oid != '')";
+ ".project_oids roid WHERE roid.oid IS NOT NULL)";
stmt.executeUpdate(sql);
stmt.close();
@ -839,10 +854,9 @@ public class PiwikStatsDB {
"SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'datasource', `timestamp`, referrer_name, agent "
+
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp " +
"WHERE process_portal_log_tmp.entity_id IS NOT NULL AND " +
"process_portal_log_tmp.entity_id != '' AND process_portal_log_tmp.entity_id " +
"WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id " +
"IN (SELECT roid.oid FROM " + ConnectDB.getStatsDBSchema()
+ ".datasource_oids roid WHERE roid.oid IS NOT NULL AND roid.oid != '')";
+ ".project_oids roid WHERE roid.oid IS NOT NULL)";
stmt.executeUpdate(sql);
stmt.close();
@ -852,10 +866,9 @@ public class PiwikStatsDB {
"SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'organization', `timestamp`, referrer_name, agent "
+
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp " +
"WHERE process_portal_log_tmp.entity_id IS NOT NULL AND " +
"process_portal_log_tmp.entity_id != '' AND process_portal_log_tmp.entity_id " +
"WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id " +
"IN (SELECT roid.oid FROM " + ConnectDB.getStatsDBSchema()
+ ".organization_oids roid WHERE roid.oid IS NOT NULL AND roid.oid != '')";
+ ".project_oids roid WHERE roid.oid IS NOT NULL)";
// stmt.executeUpdate(sql);
stmt.close();
@ -865,10 +878,9 @@ public class PiwikStatsDB {
"SELECT DISTINCT source, id_visit, country, action, url, entity_id, 'project', `timestamp`, referrer_name, agent "
+
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp " +
"WHERE process_portal_log_tmp.entity_id IS NOT NULL AND " +
"process_portal_log_tmp.entity_id != '' AND process_portal_log_tmp.entity_id " +
"WHERE process_portal_log_tmp.entity_id IS NOT NULL AND process_portal_log_tmp.entity_id " +
"IN (SELECT roid.oid FROM " + ConnectDB.getStatsDBSchema()
+ ".project_oids roid WHERE roid.oid IS NOT NULL AND roid.oid != '')";
+ ".project_oids roid WHERE roid.oid IS NOT NULL)";
stmt.executeUpdate(sql);
stmt.close();