More progress on adding queries to the code. Initial database and table creation seems OK. Downloading logs from available piwik_ids

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java

@@ -43,7 +43,7 @@ public abstract class ConnectDB {
 //		Class.forName(properties.getProperty("Stats_db_Driver"));
 
 		dbURL = "jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1";
-		usageStatsDBSchema = "usagestats";
+		usageStatsDBSchema = "usagestats_20200902";
 		statsDBSchema = "openaire_prod_stats_shadow_20200821";
 
 		Class.forName("org.apache.hive.jdbc.HiveDriver");

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java

@@ -39,6 +39,7 @@ public class PiwikStatsDB {
 	public PiwikStatsDB(String logRepoPath, String logPortalPath) throws Exception {
 		this.logRepoPath = logRepoPath;
 		this.logPortalPath = logPortalPath;
+		this.createDatabase();
 		this.createTables();
 		// The piwiklog table is not needed since it is built
 		// on top of JSON files
@@ -69,13 +70,24 @@ public class PiwikStatsDB {
 		this.CounterRobotsURL = CounterRobotsURL;
 	}
 
+	private void createDatabase() throws Exception {
+		try {
+			stmt = ConnectDB.getConnection().createStatement();
+			String createDatabase = "CREATE DATABASE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema();
+			stmt.executeUpdate(createDatabase);
+
+		} catch (Exception e) {
+			log.error("Failed to create database: " + e);
+			throw new Exception("Failed to create database: " + e.toString(), e);
+		}
+	}
+
 	private void createTables() throws Exception {
 		try {
 			stmt = ConnectDB.getConnection().createStatement();
 
 			// Create Piwiklog table - This table should exist
-			String sqlCreateTablePiwikLog =
+			String sqlCreateTablePiwikLog = "CREATE TABLE IF NOT EXISTS "
-					"CREATE TABLE IF NOT EXISTS"
 				+ ConnectDB.getUsageStatsDBSchema()
 				+ ".piwiklog(source INT, id_visit STRING, country STRING, action STRING, url STRING, "
 				+ "entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING) "
@@ -87,8 +99,7 @@ public class PiwikStatsDB {
 			// Rule for duplicate inserts @ piwiklog
 			/////////////////////////////////////////
 
-			String sqlCreateTablePortalLog = 
+			String sqlCreateTablePortalLog = "CREATE TABLE IF NOT EXISTS "
-					"CREATE TABLE IF NOT EXISTS "
 				+ ConnectDB.getUsageStatsDBSchema()
 				+ ".process_portal_log(source INT, id_visit STRING, country STRING, action STRING, url STRING, "
 				+ "entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING) "
@@ -112,8 +123,7 @@ public class PiwikStatsDB {
 	private void createTmpTables() throws Exception {
 		try {
 			Statement stmt = ConnectDB.getConnection().createStatement();
-			String sqlCreateTmpTablePiwikLog = 
+			String sqlCreateTmpTablePiwikLog = "CREATE TABLE IF NOT EXISTS "
-					"CREATE TABLE IF NOT EXISTS "
 				+ ConnectDB.getUsageStatsDBSchema()
 				+ ".piwiklogtmp(source INT, id_visit STRING, country STRING, action STRING, url STRING, entity_id STRING, "
 				+ "source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING) "
@@ -131,10 +141,7 @@ public class PiwikStatsDB {
 			// String sqlCopyPublicPiwiklog="insert into piwiklog select * from public.piwiklog;";
 			// stmt.executeUpdate(sqlCopyPublicPiwiklog);
 
-			
+			String sqlCreateTmpTablePortalLog = "CREATE TABLE IF NOT EXISTS "
-			
-			String sqlCreateTmpTablePortalLog = 
-					"CREATE TABLE IF NOT EXISTS "
 				+ ConnectDB.getUsageStatsDBSchema()
 				+ ".process_portal_log_tmp(source INT, id_visit STRING, country STRING, action STRING, url STRING, "
 				+ "entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING) "
@@ -200,8 +207,7 @@ public class PiwikStatsDB {
 		Statement stmt = ConnectDB.getConnection().createStatement();
 		ConnectDB.getConnection().setAutoCommit(false);
 
-		String stm_piwiklogtmp_json = 
+		String stm_piwiklogtmp_json = "CREATE EXTERNAL TABLE IF NOT EXISTS " +
-				"CREATE EXTERNAL TABLE IF NOT EXISTS " +
 			ConnectDB.getUsageStatsDBSchema() +
 			".piwiklogtmp_json(\n" +
 			"	`idSite` STRING,\n" +
@@ -228,15 +234,12 @@ public class PiwikStatsDB {
 			"";
 		stmt.executeUpdate(stm_piwiklogtmp_json);
 
-		
+		String stm_piwiklogtmp = "CREATE TABLE " +
-		String stm_piwiklogtmp = 
-				"CREATE TABLE " +
 			ConnectDB.getUsageStatsDBSchema() +
 			".piwiklogtmp (source BIGINT, id_Visit STRING, country STRING, action STRING, url STRING, " +
 			"entity_id STRING, source_item_type STRING, timestamp STRING, referrer_name STRING, agent STRING)  " +
 			"clustered by (source) into 100 buckets stored as orc tblproperties('transactional'='true');";
-		stmt.executeUpdate(processRepositoryLog);
+		stmt.executeUpdate(stm_piwiklogtmp);
-		
 
 		stmt.close();
 

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java

@@ -27,13 +27,13 @@ public class UsageStatsExporter {
 
 		String matomoAuthToken = "703bd17d845acdaf795e01bb1e0895b9";
 		String matomoBaseURL = "analytics.openaire.eu";
-		String repoLogPath = "/user/spyros/logs/usage_stats_logs/Repologs";
+		String repoLogPath = "/user/spyros/logs/usage_stats_logs2/Repologs";
-		String portalLogPath = "/user/spyros/logs/usage_stats_logs/Portallogs/";
+		String portalLogPath = "/user/spyros/logs/usage_stats_logs2/Portallogs/";
 		String portalMatomoID = "109";
 		String irusUKBaseURL = "https://irus.jisc.ac.uk/api/sushilite/v1_7/";
 
-		String irusUKReportPath = "/user/spyros/logs/usage_stats_logs/irusUKReports";
+		String irusUKReportPath = "/user/spyros/logs/usage_stats_logs2/irusUKReports";
-		String sarcsReportPath = "/user/spyros/logs/usage_stats_logs/sarcReports";
+		String sarcsReportPath = "/user/spyros/logs/usage_stats_logs2/sarcReports";
 
 		// connect to DB
 		ConnectDB.init(properties);