diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java index 56610fbab..981de5de1 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java @@ -37,6 +37,8 @@ public class ExecuteWorkflow { static String usageStatsDBSchema; static String statsDBSchema; static boolean recreateDbAndTables; + + static boolean piwikEmptyDirs; static boolean downloadPiwikLogs; static boolean processPiwikLogs; @@ -45,6 +47,7 @@ public class ExecuteWorkflow { static int numberOfPiwikIdsToDownload; static int numberOfSiteIdsToDownload; + static boolean laReferenciaEmptyDirs; static boolean downloadLaReferenciaLogs; static boolean processLaReferenciaLogs; @@ -98,6 +101,11 @@ public class ExecuteWorkflow { else recreateDbAndTables = false; + if (parser.get("piwikEmptyDirs").toLowerCase().equals("true")) + piwikEmptyDirs = true; + else + piwikEmptyDirs = false; + if (parser.get("downloadPiwikLogs").toLowerCase().equals("true")) downloadPiwikLogs = true; else @@ -119,6 +127,11 @@ public class ExecuteWorkflow { numberOfPiwikIdsToDownload = Integer.parseInt(parser.get("numberOfPiwikIdsToDownload")); numberOfSiteIdsToDownload = Integer.parseInt(parser.get("numberOfSiteIdsToDownload")); + if (parser.get("laReferenciaEmptyDirs").toLowerCase().equals("true")) + laReferenciaEmptyDirs = true; + else + laReferenciaEmptyDirs = false; + if (parser.get("downloadLaReferenciaLogs").toLowerCase().equals("true")) downloadLaReferenciaLogs = true; else diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java index cc5c06f5d..63df64ac4 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java @@ -39,6 +39,22 @@ public class PiwikStatsDB { } + public void reCreateLogDirs() throws IllegalArgumentException, IOException { + FileSystem dfs = FileSystem.get(new Configuration()); + + logger.info("Deleting repoLog directory: " + ExecuteWorkflow.repoLogPath); + dfs.delete(new Path(ExecuteWorkflow.repoLogPath), true); + + logger.info("Deleting portalLog directory: " + ExecuteWorkflow.portalLogPath); + dfs.delete(new Path(ExecuteWorkflow.portalLogPath), true); + + logger.info("Creating repoLog directory: " + ExecuteWorkflow.repoLogPath); + dfs.mkdirs(new Path(ExecuteWorkflow.repoLogPath)); + + logger.info("Creating portalLog directory: " + ExecuteWorkflow.portalLogPath); + dfs.mkdirs(new Path(ExecuteWorkflow.portalLogPath)); + } + public void recreateDBAndTables() throws Exception { this.createDatabase(); this.createTables(); diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java index c9883d0f7..1c6035543 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java @@ -64,11 +64,13 @@ public class UsageStatsExporter { logger.info("Initializing the download logs module"); PiwikDownloadLogs piwd = new PiwikDownloadLogs(ExecuteWorkflow.matomoBaseURL, ExecuteWorkflow.matomoAuthToken); + if (ExecuteWorkflow.piwikEmptyDirs) { + logger.info("Recreating Piwik log directories"); + piwikstatsdb.reCreateLogDirs(); + } + // Downloading piwik logs (also managing directory creation) if (ExecuteWorkflow.downloadPiwikLogs) { - logger.info("Recreating log directories"); - reCreateLogDirs(); - logger.info("Downloading piwik logs"); piwd .GetOpenAIRELogs( @@ -90,6 +92,11 @@ public class UsageStatsExporter { LaReferenciaDownloadLogs lrf = new LaReferenciaDownloadLogs(ExecuteWorkflow.lareferenciaBaseURL, ExecuteWorkflow.lareferenciaAuthToken); + if (ExecuteWorkflow.laReferenciaEmptyDirs) { + logger.info("Recreating LaReferencia log directories"); + piwikstatsdb.reCreateLogDirs(); + } + if (ExecuteWorkflow.downloadLaReferenciaLogs) { logger.info("Downloading LaReferencia logs"); lrf.GetLaReferenciaRepos(ExecuteWorkflow.lareferenciaLogPath); diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json index cc621a958..3d282a7a7 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json +++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json @@ -101,6 +101,12 @@ "paramDescription": "Re-create database and initial tables?", "paramRequired": true }, + { + "paramName": "pwed", + "paramLongName": "piwikEmptyDirs", + "paramDescription": "Empty piwik directories?", + "paramRequired": true + }, { "paramName": "ppwl", "paramLongName": "processPiwikLogs", @@ -137,6 +143,12 @@ "paramDescription": "Limit the number of the downloaded siteids (La Referencia logs) to the first numberOfSiteIdsToDownload", "paramRequired": true }, + { + "paramName": "lerd", + "paramLongName": "laReferenciaEmptyDirs", + "paramDescription": "Empty LaReferencia directories?", + "paramRequired": true + }, { "paramName": "plrl", "paramLongName": "processLaReferenciaLogs", diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml index ac1514ef2..2ad3e6669 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml +++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml @@ -59,12 +59,14 @@ --usageStatsDBSchema${usageStatsDBSchema} --statsDBSchema${statsDBSchema} --recreateDbAndTables${recreateDbAndTables} + --piwikEmptyDirs${piwikEmptyDirs} --downloadPiwikLogs${downloadPiwikLogs} --processPiwikLogs${processPiwikLogs} --startingLogPeriod${startingLogPeriod} --endingLogPeriod${endingLogPeriod} --numberOfPiwikIdsToDownload${numberOfPiwikIdsToDownload} --numberOfSiteIdsToDownload${numberOfSiteIdsToDownload} + --laReferenciaEmptyDirs${laReferenciaEmptyDirs} --downloadLaReferenciaLogs${downloadLaReferenciaLogs} --processLaReferenciaLogs${processLaReferenciaLogs} --irusCreateTablesEmptyDirs${irusCreateTablesEmptyDirs}