diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
index 56610fbab..981de5de1 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
@@ -37,6 +37,8 @@ public class ExecuteWorkflow {
static String usageStatsDBSchema;
static String statsDBSchema;
static boolean recreateDbAndTables;
+
+ static boolean piwikEmptyDirs;
static boolean downloadPiwikLogs;
static boolean processPiwikLogs;
@@ -45,6 +47,7 @@ public class ExecuteWorkflow {
static int numberOfPiwikIdsToDownload;
static int numberOfSiteIdsToDownload;
+ static boolean laReferenciaEmptyDirs;
static boolean downloadLaReferenciaLogs;
static boolean processLaReferenciaLogs;
@@ -98,6 +101,11 @@ public class ExecuteWorkflow {
else
recreateDbAndTables = false;
+ if (parser.get("piwikEmptyDirs").toLowerCase().equals("true"))
+ piwikEmptyDirs = true;
+ else
+ piwikEmptyDirs = false;
+
if (parser.get("downloadPiwikLogs").toLowerCase().equals("true"))
downloadPiwikLogs = true;
else
@@ -119,6 +127,11 @@ public class ExecuteWorkflow {
numberOfPiwikIdsToDownload = Integer.parseInt(parser.get("numberOfPiwikIdsToDownload"));
numberOfSiteIdsToDownload = Integer.parseInt(parser.get("numberOfSiteIdsToDownload"));
+ if (parser.get("laReferenciaEmptyDirs").toLowerCase().equals("true"))
+ laReferenciaEmptyDirs = true;
+ else
+ laReferenciaEmptyDirs = false;
+
if (parser.get("downloadLaReferenciaLogs").toLowerCase().equals("true"))
downloadLaReferenciaLogs = true;
else
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
index cc5c06f5d..63df64ac4 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/PiwikStatsDB.java
@@ -39,6 +39,22 @@ public class PiwikStatsDB {
}
+ public void reCreateLogDirs() throws IllegalArgumentException, IOException {
+ FileSystem dfs = FileSystem.get(new Configuration());
+
+ logger.info("Deleting repoLog directory: " + ExecuteWorkflow.repoLogPath);
+ dfs.delete(new Path(ExecuteWorkflow.repoLogPath), true);
+
+ logger.info("Deleting portalLog directory: " + ExecuteWorkflow.portalLogPath);
+ dfs.delete(new Path(ExecuteWorkflow.portalLogPath), true);
+
+ logger.info("Creating repoLog directory: " + ExecuteWorkflow.repoLogPath);
+ dfs.mkdirs(new Path(ExecuteWorkflow.repoLogPath));
+
+ logger.info("Creating portalLog directory: " + ExecuteWorkflow.portalLogPath);
+ dfs.mkdirs(new Path(ExecuteWorkflow.portalLogPath));
+ }
+
public void recreateDBAndTables() throws Exception {
this.createDatabase();
this.createTables();
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
index c9883d0f7..1c6035543 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
@@ -64,11 +64,13 @@ public class UsageStatsExporter {
logger.info("Initializing the download logs module");
PiwikDownloadLogs piwd = new PiwikDownloadLogs(ExecuteWorkflow.matomoBaseURL, ExecuteWorkflow.matomoAuthToken);
+ if (ExecuteWorkflow.piwikEmptyDirs) {
+ logger.info("Recreating Piwik log directories");
+ piwikstatsdb.reCreateLogDirs();
+ }
+
// Downloading piwik logs (also managing directory creation)
if (ExecuteWorkflow.downloadPiwikLogs) {
- logger.info("Recreating log directories");
- reCreateLogDirs();
-
logger.info("Downloading piwik logs");
piwd
.GetOpenAIRELogs(
@@ -90,6 +92,11 @@ public class UsageStatsExporter {
LaReferenciaDownloadLogs lrf = new LaReferenciaDownloadLogs(ExecuteWorkflow.lareferenciaBaseURL,
ExecuteWorkflow.lareferenciaAuthToken);
+ if (ExecuteWorkflow.laReferenciaEmptyDirs) {
+ logger.info("Recreating LaReferencia log directories");
+ piwikstatsdb.reCreateLogDirs();
+ }
+
if (ExecuteWorkflow.downloadLaReferenciaLogs) {
logger.info("Downloading LaReferencia logs");
lrf.GetLaReferenciaRepos(ExecuteWorkflow.lareferenciaLogPath);
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
index cc621a958..3d282a7a7 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
@@ -101,6 +101,12 @@
"paramDescription": "Re-create database and initial tables?",
"paramRequired": true
},
+ {
+ "paramName": "pwed",
+ "paramLongName": "piwikEmptyDirs",
+ "paramDescription": "Empty piwik directories?",
+ "paramRequired": true
+ },
{
"paramName": "ppwl",
"paramLongName": "processPiwikLogs",
@@ -137,6 +143,12 @@
"paramDescription": "Limit the number of the downloaded siteids (La Referencia logs) to the first numberOfSiteIdsToDownload",
"paramRequired": true
},
+ {
+ "paramName": "lerd",
+ "paramLongName": "laReferenciaEmptyDirs",
+ "paramDescription": "Empty LaReferencia directories?",
+ "paramRequired": true
+ },
{
"paramName": "plrl",
"paramLongName": "processLaReferenciaLogs",
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
index ac1514ef2..2ad3e6669 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
@@ -59,12 +59,14 @@
--usageStatsDBSchema${usageStatsDBSchema}
--statsDBSchema${statsDBSchema}
--recreateDbAndTables${recreateDbAndTables}
+ --piwikEmptyDirs${piwikEmptyDirs}
--downloadPiwikLogs${downloadPiwikLogs}
--processPiwikLogs${processPiwikLogs}
--startingLogPeriod${startingLogPeriod}
--endingLogPeriod${endingLogPeriod}
--numberOfPiwikIdsToDownload${numberOfPiwikIdsToDownload}
--numberOfSiteIdsToDownload${numberOfSiteIdsToDownload}
+ --laReferenciaEmptyDirs${laReferenciaEmptyDirs}
--downloadLaReferenciaLogs${downloadLaReferenciaLogs}
--processLaReferenciaLogs${processLaReferenciaLogs}
--irusCreateTablesEmptyDirs${irusCreateTablesEmptyDirs}