From 5af2abbea517fe26343ccaadcf126608a58e5f2c Mon Sep 17 00:00:00 2001 From: Spyros Zoupanos Date: Fri, 4 Sep 2020 19:49:07 +0300 Subject: [PATCH] Moving variable declarations to a more appropriate place, adding drop table code --- .../usagestats/export/PiwikDownloadLogs.java | 3 +-- .../graph/usagestats/export/PiwikStatsDB.java | 18 +++++++++++---- .../usagestats/export/UsageStatsExporter.java | 22 +++++++++---------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikDownloadLogs.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikDownloadLogs.java index e2bbea8691..9e4a3eb34d 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikDownloadLogs.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikDownloadLogs.java @@ -58,7 +58,6 @@ public class PiwikDownloadLogs { try { System.out.println("===> Connecting to: " + url); URL website = new URL(url); - System.out.println("Connection url -----> " + url); URLConnection connection = website.openConnection(); // connection.setRequestProperty ("Authorization", "Basic "+encoded); @@ -72,7 +71,7 @@ public class PiwikDownloadLogs { } } - System.out.println("response ====> " + response.toString()); +// System.out.println("response ====> " + response.toString()); return response.toString(); } catch (Exception e) { diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java index 3a67adda0f..378001fb60 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java @@ -204,9 +204,15 @@ public class PiwikStatsDB { // } public void processRepositoryLog() throws Exception { + Statement stmt = ConnectDB.getConnection().createStatement(); ConnectDB.getConnection().setAutoCommit(false); + String drop_piwiklogtmp_json = "DROP TABLE IF EXISTS " + + ConnectDB.getUsageStatsDBSchema() + + ".piwiklogtmp_json"; + stmt.executeUpdate(drop_piwiklogtmp_json); + String stm_piwiklogtmp_json = "CREATE EXTERNAL TABLE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp_json(\n" + @@ -229,16 +235,20 @@ public class PiwikStatsDB { " >\n" + ")\n" + "ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'\n" + - "LOCATION '/user/spyros/logs/usage_stats_logs/Repologs2/'\n" + - "TBLPROPERTIES (\"transactional\"=\"false\");\n" + - ""; + "LOCATION '" + UsageStatsExporter.repoLogPath + "'\n" + + "TBLPROPERTIES (\"transactional\"=\"false\")"; stmt.executeUpdate(stm_piwiklogtmp_json); + String drop_piwiklogtmp = "DROP TABLE IF EXISTS " + + ConnectDB.getUsageStatsDBSchema() + + ".piwiklogtmp"; + stmt.executeUpdate(drop_piwiklogtmp); + String stm_piwiklogtmp = "CREATE TABLE " + ConnectDB.getUsageStatsDBSchema() + ".piwiklogtmp (source BIGINT, id_Visit STRING, country STRING, action STRING, url STRING, " + "entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING) " + - "clustered by (source) into 100 buckets stored as orc tblproperties('transactional'='true');"; + "clustered by (source) into 100 buckets stored as orc tblproperties('transactional'='true')"; stmt.executeUpdate(stm_piwiklogtmp); stmt.close(); diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java index 82d1519b41..bc109a8558 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java @@ -10,6 +10,16 @@ public class UsageStatsExporter { private Logger log = Logger.getLogger(this.getClass()); private Properties properties; + static String matomoAuthToken = "703bd17d845acdaf795e01bb1e0895b9"; + static String matomoBaseURL = "analytics.openaire.eu"; + static String repoLogPath = "/user/spyros/logs/usage_stats_logs2/Repologs"; + static String portalLogPath = "/user/spyros/logs/usage_stats_logs2/Portallogs/"; + static String portalMatomoID = "109"; + static String irusUKBaseURL = "https://irus.jisc.ac.uk/api/sushilite/v1_7/"; + + static String irusUKReportPath = "/user/spyros/logs/usage_stats_logs2/irusUKReports"; + static String sarcsReportPath = "/user/spyros/logs/usage_stats_logs2/sarcReports"; + public UsageStatsExporter(Properties properties) { this.properties = properties; } @@ -25,16 +35,6 @@ public class UsageStatsExporter { // String portalMatomoID = properties.getProperty("portal_MatomoID"); // String irusUKBaseURL = properties.getProperty("IRUS_UK_BaseUrl"); - String matomoAuthToken = "703bd17d845acdaf795e01bb1e0895b9"; - String matomoBaseURL = "analytics.openaire.eu"; - String repoLogPath = "/user/spyros/logs/usage_stats_logs2/Repologs"; - String portalLogPath = "/user/spyros/logs/usage_stats_logs2/Portallogs/"; - String portalMatomoID = "109"; - String irusUKBaseURL = "https://irus.jisc.ac.uk/api/sushilite/v1_7/"; - - String irusUKReportPath = "/user/spyros/logs/usage_stats_logs2/irusUKReports"; - String sarcsReportPath = "/user/spyros/logs/usage_stats_logs2/sarcReports"; - // connect to DB ConnectDB.init(properties); @@ -44,7 +44,7 @@ public class UsageStatsExporter { // // Download the statistics - The following 2 lines are not needed after the download - Commenting them out for // // the moment PiwikDownloadLogs piwd = new PiwikDownloadLogs(matomoBaseURL, matomoAuthToken); - piwd.GetOpenAIRELogs(repoLogPath, portalLogPath, portalMatomoID); +// piwd.GetOpenAIRELogs(repoLogPath, portalLogPath, portalMatomoID); // Create DB tables, insert/update statistics // String cRobotsUrl = properties.getProperty("COUNTER_robots_Url");