diff --git a/dhp-workflows/dhp-usage-stats-update/job-override.properties.bk b/dhp-workflows/dhp-usage-stats-update/job-override.properties.bk
deleted file mode 100644
index 67d872e9d..000000000
--- a/dhp-workflows/dhp-usage-stats-update/job-override.properties.bk
+++ /dev/null
@@ -1,7 +0,0 @@
-stats_db_name=openaire_beta_20200618_stats
-openaire_db_name=openaire_beta_20200618
-external_stats_db_name=stats_ext
-stats_db_shadow_name=openaire_beta_20200618_stats_shadow
-hive_timeout=3000
-hive_spark_client_timeout=100000
-hive_spark_client_server_timeout=100000
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java
index ff98963a4..d8ca38618 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java
@@ -7,11 +7,8 @@
package eu.dnetlib.oa.graph.usagestats.export;
/**
- *
* @author D. Pierrakos, S. Zoupanos
- *
*/
-
/**
* @author D. Pierrakos, S. Zoupanos
*/
@@ -30,18 +27,16 @@ public abstract class ConnectDB {
private static String dbHiveUrl;
private static String dbImpalaUrl;
- private static String dbUsername;
- private static String dbPassword;
private static String usageStatsDBSchema;
private static String statsDBSchema;
private final static Logger log = Logger.getLogger(ConnectDB.class);
- static void init(Properties properties) throws ClassNotFoundException {
+ static void init() throws ClassNotFoundException {
- dbHiveUrl = "jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1";
- dbImpalaUrl = "jdbc:hive2://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;auth=noSasl";
- usageStatsDBSchema = "usagestats_20200913";
- statsDBSchema = "openaire_prod_stats_shadow_20200821";
+ dbHiveUrl = ExecuteWorkflow.dbHiveUrl;
+ dbImpalaUrl = ExecuteWorkflow.dbImpalaUrl;
+ usageStatsDBSchema = ExecuteWorkflow.usageStatsDBSchema;
+ statsDBSchema = ExecuteWorkflow.statsDBSchema;
Class.forName("org.apache.hive.jdbc.HiveDriver");
}
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
index 7afe760ee..e0dc4e00e 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
@@ -27,6 +27,10 @@ public class ExecuteWorkflow {
static String lareferenciaLogPath;
static String lareferenciaBaseURL;
static String lareferenciaAuthToken;
+ static String dbHiveUrl;
+ static String dbImpalaUrl;
+ static String usageStatsDBSchema;
+ static String statsDBSchema;
public static void main(String args[]) throws Exception {
@@ -52,6 +56,11 @@ public class ExecuteWorkflow {
lareferenciaBaseURL = parser.get("lareferenciaBaseURL");
lareferenciaAuthToken = parser.get("lareferenciaAuthToken");
+ dbHiveUrl = parser.get("dbHiveUrl");
+ dbImpalaUrl = parser.get("dbImpalaUrl");
+ usageStatsDBSchema = parser.get("usageStatsDBSchema");
+ statsDBSchema = parser.get("statsDBSchema");
+
System.out.println("====> Printing parsed variables");
System.out.println(ExecuteWorkflow.matomoAuthToken);
System.out.println(ExecuteWorkflow.matomoBaseURL);
@@ -64,6 +73,10 @@ public class ExecuteWorkflow {
System.out.println(ExecuteWorkflow.lareferenciaLogPath);
System.out.println(ExecuteWorkflow.lareferenciaBaseURL);
System.out.println(ExecuteWorkflow.lareferenciaAuthToken);
+ System.out.println(ExecuteWorkflow.dbHiveUrl);
+ System.out.println(ExecuteWorkflow.dbImpalaUrl);
+ System.out.println(ExecuteWorkflow.usageStatsDBSchema);
+ System.out.println(ExecuteWorkflow.statsDBSchema);
UsageStatsExporter usagestatsExport = new UsageStatsExporter();
usagestatsExport.export();
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
index 0e731d03a..4cdf11310 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
@@ -78,6 +78,12 @@ public class PiwikStatsDB {
private void createDatabase() throws Exception {
try {
stmt = ConnectDB.getHiveConnection().createStatement();
+
+ logger.info("Dropping usagestats DB");
+ String dropDatabase = "DROP DATABASE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + "CASCADE;";
+ stmt.executeUpdate(dropDatabase);
+
+ logger.info("Creating usagestats DB");
String createDatabase = "CREATE DATABASE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema();
stmt.executeUpdate(createDatabase);
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java
index 18a376a60..1708a1c64 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java
@@ -7,11 +7,8 @@
package eu.dnetlib.oa.graph.usagestats.export;
/**
- *
* @author D. Pierrakos, S. Zoupanos
- *
*/
-
/**
* @author D. Pierrakos, S. Zoupanos
*/
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
index ba709ec98..50529a06f 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
@@ -43,12 +43,18 @@ public class UsageStatsExporter {
public void export() throws Exception {
+ logger.info("Initialising DB properties");
+ ConnectDB.init();
+
+// System.exit(0);
+
// runImpalaQuery();
// Create DB tables - they are also needed to download the statistics too
logger.info("Creating database and tables");
PiwikStatsDB piwikstatsdb = new PiwikStatsDB(ExecuteWorkflow.repoLogPath, ExecuteWorkflow.portalLogPath);
//
+
// // Download the statistics - The following 2 lines are not needed after the download - Commenting them out for
// // the moment
logger.info("Initializing the download logs module");
@@ -88,7 +94,7 @@ public class UsageStatsExporter {
// log.info("sarc done");
// // finalize usagestats
-// piwikstatsdb.finalizeStats();
+ piwikstatsdb.finalizeStats();
// log.info("finalized stats");
}
}
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
index 5cb89a845..a8737750e 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
@@ -70,9 +70,29 @@
"paramLongName": "lareferenciaAuthToken",
"paramDescription": "activate tranform-only mode. Only apply transformation step",
"paramRequired": true
+ },
+ {
+ "paramName": "dbhu",
+ "paramLongName": "dbHiveUrl",
+ "paramDescription": "activate tranform-only mode. Only apply transformation step",
+ "paramRequired": true
+ },
+ {
+ "paramName": "dbiu",
+ "paramLongName": "dbImpalaUrl",
+ "paramDescription": "activate tranform-only mode. Only apply transformation step",
+ "paramRequired": true
+ },
+ {
+ "paramName": "usdbs",
+ "paramLongName": "usageStatsDBSchema",
+ "paramDescription": "activate tranform-only mode. Only apply transformation step",
+ "paramRequired": true
+ },
+ {
+ "paramName": "sdbs",
+ "paramLongName": "statsDBSchema",
+ "paramDescription": "activate tranform-only mode. Only apply transformation step",
+ "paramRequired": true
}
]
-
-
-
-
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml
index d369804b6..b5c807378 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml
@@ -21,7 +21,11 @@
hiveJdbcUrl
- jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000
+ jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1
+
+
+ impalaJdbcUrl
+ jdbc:hive2://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;auth=noSasl;
oozie.wf.workflow.notification.url
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
index df349e49e..8cd425850 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
@@ -2,11 +2,15 @@
hiveMetastoreUris
- hive server metastore URIs
+ Hive server metastore URIs
hiveJdbcUrl
- hive server jdbc url
+ Hive server jdbc url
+
+
+ impalaJdbcUrl
+ Impala server jdbc url
@@ -50,6 +54,10 @@
--lareferenciaLogPath${lareferenciaLogPath}
--lareferenciaBaseURL${lareferenciaBaseURL}
--lareferenciaAuthToken${lareferenciaAuthToken}
+ --dbHiveUrl${hiveJdbcUrl}
+ --dbImpalaUrl${impalaJdbcUrl}
+ --usageStatsDBSchema${usageStatsDBSchema}
+ --statsDBSchema${statsDBSchema}