diff --git a/dhp-workflows/dhp-usage-stats-update/job-override.properties.bk b/dhp-workflows/dhp-usage-stats-update/job-override.properties.bk deleted file mode 100644 index 67d872e9d..000000000 --- a/dhp-workflows/dhp-usage-stats-update/job-override.properties.bk +++ /dev/null @@ -1,7 +0,0 @@ -stats_db_name=openaire_beta_20200618_stats -openaire_db_name=openaire_beta_20200618 -external_stats_db_name=stats_ext -stats_db_shadow_name=openaire_beta_20200618_stats_shadow -hive_timeout=3000 -hive_spark_client_timeout=100000 -hive_spark_client_server_timeout=100000 diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java index ff98963a4..d8ca38618 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ConnectDB.java @@ -7,11 +7,8 @@ package eu.dnetlib.oa.graph.usagestats.export; /** - * * @author D. Pierrakos, S. Zoupanos - * */ - /** * @author D. Pierrakos, S. Zoupanos */ @@ -30,18 +27,16 @@ public abstract class ConnectDB { private static String dbHiveUrl; private static String dbImpalaUrl; - private static String dbUsername; - private static String dbPassword; private static String usageStatsDBSchema; private static String statsDBSchema; private final static Logger log = Logger.getLogger(ConnectDB.class); - static void init(Properties properties) throws ClassNotFoundException { + static void init() throws ClassNotFoundException { - dbHiveUrl = "jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1"; - dbImpalaUrl = "jdbc:hive2://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;auth=noSasl"; - usageStatsDBSchema = "usagestats_20200913"; - statsDBSchema = "openaire_prod_stats_shadow_20200821"; + dbHiveUrl = ExecuteWorkflow.dbHiveUrl; + dbImpalaUrl = ExecuteWorkflow.dbImpalaUrl; + usageStatsDBSchema = ExecuteWorkflow.usageStatsDBSchema; + statsDBSchema = ExecuteWorkflow.statsDBSchema; Class.forName("org.apache.hive.jdbc.HiveDriver"); } diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java index 7afe760ee..e0dc4e00e 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java @@ -27,6 +27,10 @@ public class ExecuteWorkflow { static String lareferenciaLogPath; static String lareferenciaBaseURL; static String lareferenciaAuthToken; + static String dbHiveUrl; + static String dbImpalaUrl; + static String usageStatsDBSchema; + static String statsDBSchema; public static void main(String args[]) throws Exception { @@ -52,6 +56,11 @@ public class ExecuteWorkflow { lareferenciaBaseURL = parser.get("lareferenciaBaseURL"); lareferenciaAuthToken = parser.get("lareferenciaAuthToken"); + dbHiveUrl = parser.get("dbHiveUrl"); + dbImpalaUrl = parser.get("dbImpalaUrl"); + usageStatsDBSchema = parser.get("usageStatsDBSchema"); + statsDBSchema = parser.get("statsDBSchema"); + System.out.println("====> Printing parsed variables"); System.out.println(ExecuteWorkflow.matomoAuthToken); System.out.println(ExecuteWorkflow.matomoBaseURL); @@ -64,6 +73,10 @@ public class ExecuteWorkflow { System.out.println(ExecuteWorkflow.lareferenciaLogPath); System.out.println(ExecuteWorkflow.lareferenciaBaseURL); System.out.println(ExecuteWorkflow.lareferenciaAuthToken); + System.out.println(ExecuteWorkflow.dbHiveUrl); + System.out.println(ExecuteWorkflow.dbImpalaUrl); + System.out.println(ExecuteWorkflow.usageStatsDBSchema); + System.out.println(ExecuteWorkflow.statsDBSchema); UsageStatsExporter usagestatsExport = new UsageStatsExporter(); usagestatsExport.export(); diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java index 0e731d03a..4cdf11310 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java @@ -78,6 +78,12 @@ public class PiwikStatsDB { private void createDatabase() throws Exception { try { stmt = ConnectDB.getHiveConnection().createStatement(); + + logger.info("Dropping usagestats DB"); + String dropDatabase = "DROP DATABASE IF EXISTS " + ConnectDB.getUsageStatsDBSchema() + "CASCADE;"; + stmt.executeUpdate(dropDatabase); + + logger.info("Creating usagestats DB"); String createDatabase = "CREATE DATABASE IF NOT EXISTS " + ConnectDB.getUsageStatsDBSchema(); stmt.executeUpdate(createDatabase); diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java index 18a376a60..1708a1c64 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ReadCounterRobotsList.java @@ -7,11 +7,8 @@ package eu.dnetlib.oa.graph.usagestats.export; /** - * * @author D. Pierrakos, S. Zoupanos - * */ - /** * @author D. Pierrakos, S. Zoupanos */ diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java index ba709ec98..50529a06f 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java @@ -43,12 +43,18 @@ public class UsageStatsExporter { public void export() throws Exception { + logger.info("Initialising DB properties"); + ConnectDB.init(); + +// System.exit(0); + // runImpalaQuery(); // Create DB tables - they are also needed to download the statistics too logger.info("Creating database and tables"); PiwikStatsDB piwikstatsdb = new PiwikStatsDB(ExecuteWorkflow.repoLogPath, ExecuteWorkflow.portalLogPath); // + // // Download the statistics - The following 2 lines are not needed after the download - Commenting them out for // // the moment logger.info("Initializing the download logs module"); @@ -88,7 +94,7 @@ public class UsageStatsExporter { // log.info("sarc done"); // // finalize usagestats -// piwikstatsdb.finalizeStats(); + piwikstatsdb.finalizeStats(); // log.info("finalized stats"); } } diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json index 5cb89a845..a8737750e 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json +++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json @@ -70,9 +70,29 @@ "paramLongName": "lareferenciaAuthToken", "paramDescription": "activate tranform-only mode. Only apply transformation step", "paramRequired": true + }, + { + "paramName": "dbhu", + "paramLongName": "dbHiveUrl", + "paramDescription": "activate tranform-only mode. Only apply transformation step", + "paramRequired": true + }, + { + "paramName": "dbiu", + "paramLongName": "dbImpalaUrl", + "paramDescription": "activate tranform-only mode. Only apply transformation step", + "paramRequired": true + }, + { + "paramName": "usdbs", + "paramLongName": "usageStatsDBSchema", + "paramDescription": "activate tranform-only mode. Only apply transformation step", + "paramRequired": true + }, + { + "paramName": "sdbs", + "paramLongName": "statsDBSchema", + "paramDescription": "activate tranform-only mode. Only apply transformation step", + "paramRequired": true } ] - - - - diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml index d369804b6..b5c807378 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml +++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/config-default.xml @@ -21,7 +21,11 @@ hiveJdbcUrl - jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000 + jdbc:hive2://iis-cdh5-test-m3.ocean.icm.edu.pl:10000/;UseNativeQuery=1 + + + impalaJdbcUrl + jdbc:hive2://iis-cdh5-test-gw.ocean.icm.edu.pl:21050/;auth=noSasl; oozie.wf.workflow.notification.url diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml index df349e49e..8cd425850 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml @@ -2,11 +2,15 @@ hiveMetastoreUris - hive server metastore URIs + Hive server metastore URIs hiveJdbcUrl - hive server jdbc url + Hive server jdbc url + + + impalaJdbcUrl + Impala server jdbc url @@ -50,6 +54,10 @@ --lareferenciaLogPath${lareferenciaLogPath} --lareferenciaBaseURL${lareferenciaBaseURL} --lareferenciaAuthToken${lareferenciaAuthToken} + --dbHiveUrl${hiveJdbcUrl} + --dbImpalaUrl${impalaJdbcUrl} + --usageStatsDBSchema${usageStatsDBSchema} + --statsDBSchema${statsDBSchema}