From a2d64b46445130849be0db0ef0fe873871f27f97 Mon Sep 17 00:00:00 2001
From: Spyros Zoupanos <spyros@zoupanos.net>
Date: Sun, 20 Sep 2020 14:03:16 +0300
Subject: [PATCH] Added lareferencialogtmp_json table creation

---
 .../usagestats/export/LaReferenciaStats.java  | 44 +++++++++++++++++++
 .../graph/usagestats/export/PiwikStatsDB.java |  2 +-
 .../usagestats/export/UsageStatsExporter.java |  2 +-
 3 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java
index 1cf0b6e91e..84b3a9ef3a 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java
@@ -123,6 +123,50 @@ public class LaReferenciaStats {
 	}
 
 	public void processlaReferenciaLog() throws Exception {
+		Statement stmt = ConnectDB.getConnection().createStatement();
+		ConnectDB.getConnection().setAutoCommit(false);
+
+		System.out.println("====> Dropping lareferencialogtmp_json table");
+		String drop_piwiklogtmp_json = "DROP TABLE IF EXISTS " +
+			ConnectDB.getUsageStatsDBSchema() +
+			".lareferencialogtmp_json";
+		stmt.executeUpdate(drop_piwiklogtmp_json);
+		System.out.println("====> Dropped lareferencialogtmp_json table");
+
+		System.out.println("====> Creating lareferencialogtmp_json");
+		String create_piwiklogtmp_json = "CREATE EXTERNAL TABLE IF NOT EXISTS " +
+			ConnectDB.getUsageStatsDBSchema() +
+			".lareferencialogtmp_json(\n" +
+			"	`idSite` STRING,\n" +
+			"	`idVisit` STRING,\n" +
+			"	`country` STRING,\n" +
+			"	`referrerName` STRING,\n" +
+			"	`browser` STRING,\n" +
+			"	`repItem` STRING,\n" +
+			"	`actionDetails` ARRAY<\n" +
+			"						struct<\n" +
+			"							timestamp: STRING,\n" +
+			"							type: STRING,\n" +
+			"							url: STRING,\n" +
+			"							`customVariables`: struct<\n" +
+			"								`1`: struct<\n" +
+			"								`customVariablePageValue1`: STRING\n" +
+			"										>,\n" +
+			"								`2`: struct<\n" +
+			"								`customVariablePageValue2`: STRING\n" +
+			"										>\n" +
+			"								>\n" +
+			"							>\n" +
+			"						>" +
+			")\n" +
+			"ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'\n" +
+			"LOCATION '" + UsageStatsExporter.lareferenciaLogPath + "'\n" +
+			"TBLPROPERTIES (\"transactional\"=\"false\")";
+		stmt.executeUpdate(create_piwiklogtmp_json);
+		System.out.println("====> Created lareferencialogtmp_json");
+	}
+
+	public void processlaReferenciaLogOld() throws Exception {
 
 		Statement stmt = ConnectDB.getConnection().createStatement();
 		ConnectDB.getConnection().setAutoCommit(false);
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
index 53161b27b3..cdd509f867 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
@@ -168,7 +168,7 @@ public class PiwikStatsDB {
 			this.robotsList = counterRobots.getRobotsPatterns();
 
 			System.out.println("====> Processing repository logs");
-//			processRepositoryLog();
+			processRepositoryLog();
 			System.out.println("====> Repository logs process done");
 			log.info("repository process done");
 
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
index c849eccc9b..1628c4ff51 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
@@ -60,7 +60,7 @@ public class UsageStatsExporter {
 		System.out.println("====> Creating LaReferencia tables");
 		LaReferenciaDownloadLogs lrf = new LaReferenciaDownloadLogs(lareferenciaBaseURL, lareferenciaAuthToken);
 		System.out.println("====> Downloading LaReferencia logs");
-		lrf.GetLaReferenciaRepos(lareferenciaLogPath);
+//		lrf.GetLaReferenciaRepos(lareferenciaLogPath);
 		System.out.println("====> Downloaded LaReferencia logs");
 		LaReferenciaStats lastats = new LaReferenciaStats(lareferenciaLogPath);
 		System.out.println("====> Processing LaReferencia logs");