spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

portalStats finished - Needs testing. Working on updateProdTables

This commit is contained in:
Spyros Zoupanos 2020-09-12 21:24:31 +03:00
parent 8ddf1dcc15
commit 9caac3e3e3
1 changed files with 47 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
View File

@ -191,14 +191,16 @@ public class PiwikStatsDB {
System.out.println("====> DownloadsStats processing starts");
System.out.println("====> Processing portal logs");
processPortalLog();
// processPortalLog();
System.out.println("====> Portal logs process done");
log.info("portal process done");
System.exit(0);
System.out.println("====> Processing portal usagestats");
portalStats();
log.info("portal usagestats done");
System.out.println("====> Portal usagestats process done");
System.exit(0);
updateProdTables();
log.info("updateProdTables done");
@ -802,34 +804,49 @@ public class PiwikStatsDB {
stmt.close();
}
public void portalStats() throws SQLException {
Connection con = ConnectDB.getConnection();
Statement stmt = con.createStatement();
con.setAutoCommit(false);
String sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'oaItem\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.result_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
System.out.println("====> PortalStats - Step 1");
String sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp " +
"SELECT DISTINCT source, id_visit, country, action, url, roid.oid, 'oaItem', timestamp, referrer_name, agent " +
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp, " +
ConnectDB.getStatsDBSchema() + ".result_oids roid " +
"WHERE entity_id IS NOT null AND entity_id=roid.oid AND roid.oid IS NOT null";
stmt.executeUpdate(sql);
stmt.close();
// con.commit();
System.out.println("====> PortalStats - Step 2");
stmt = con.createStatement();
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'datasource\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.datasource_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp " +
"SELECT DISTINCT source, id_visit, country, action, url, roid.oid, 'datasource', timestamp, referrer_name, agent " +
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp, " +
ConnectDB.getStatsDBSchema() + ".datasource_oids roid " +
"WHERE entity_id IS NOT null AND entity_id=roid.oid AND roid.oid IS NOT null";
stmt.executeUpdate(sql);
stmt.close();
// con.commit();
System.out.println("====> PortalStats - Step 3");
stmt = con.createStatement();
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'organization\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.organization_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp " +
"SELECT DISTINCT source, id_visit, country, action, url, roid.oid, 'organization', timestamp, referrer_name, agent " +
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp, " +
ConnectDB.getStatsDBSchema() + ".datasource_oids roid " +
"WHERE entity_id IS NOT null AND entity_id=roid.oid AND roid.oid IS NOT null";
stmt.executeUpdate(sql);
stmt.close();
// con.commit();
System.out.println("====> PortalStats - Step 4");
stmt = con.createStatement();
sql = "INSERT INTO piwiklogtmp SELECT DISTINCT source, id_visit, country, action, url, roid.orid, \'project\', timestamp, referrer_name, agent FROM process_portal_log_tmp, public.project_oids roid WHERE entity_id IS NOT null AND entity_id=roid.orid AND roid.orid IS NOT null;";
sql = "INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp " +
"SELECT DISTINCT source, id_visit, country, action, url, roid.oid, 'project', timestamp, referrer_name, agent " +
"FROM " + ConnectDB.getUsageStatsDBSchema() + ".process_portal_log_tmp, " +
ConnectDB.getStatsDBSchema() + ".project_oids roid " +
"WHERE entity_id IS NOT null AND entity_id=roid.oid AND roid.oid IS NOT null";
stmt.executeUpdate(sql);
stmt.close();
// con.commit();
con.close();
}
@ -1116,19 +1133,32 @@ public class PiwikStatsDB {
Statement stmt = ConnectDB.getConnection().createStatement();
ConnectDB.getConnection().setAutoCommit(false);
String sql = "insert into piwiklog select * from piwiklogtmp;";
// String sql = "insert into piwiklog select * from piwiklogtmp;";
String sql =
"INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".piwiklog " +
"SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp";
stmt.executeUpdate(sql);
sql = "insert into views_stats select * from views_stats_tmp;";
// sql = "insert into views_stats select * from views_stats_tmp;";
sql =
"INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".views_stats " +
"SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".views_stats_tmp";
stmt.executeUpdate(sql);
sql = "insert into downloads_stats select * from downloads_stats_tmp;";
// sql = "insert into downloads_stats select * from downloads_stats_tmp;";
sql =
"INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats " +
"SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".downloads_stats_tmp";
stmt.executeUpdate(sql);
sql = "insert into pageviews_stats select * from pageviews_stats_tmp;";
// sql = "insert into pageviews_stats select * from pageviews_stats_tmp;";
sql =
"INSERT INTO " + ConnectDB.getUsageStatsDBSchema() + ".pageviews_stats " +
"SELECT * FROM " + ConnectDB.getUsageStatsDBSchema() + ".pageviews_stats_tmp";
stmt.executeUpdate(sql);
sql = "DROP TABLE IF EXISTS views_stats_tmp;";
// sql = "DROP TABLE IF EXISTS views_stats_tmp;";
sql = "";
stmt.executeUpdate(sql);
sql = "DROP TABLE IF EXISTS downloads_stats_tmp;";