forked from D-Net/dnet-hadoop
More control on directory cleanup
This commit is contained in:
parent
8f24a6388e
commit
a1dc930486
|
@ -37,6 +37,8 @@ public class ExecuteWorkflow {
|
|||
static String usageStatsDBSchema;
|
||||
static String statsDBSchema;
|
||||
static boolean recreateDbAndTables;
|
||||
|
||||
static boolean piwikEmptyDirs;
|
||||
static boolean downloadPiwikLogs;
|
||||
static boolean processPiwikLogs;
|
||||
|
||||
|
@ -45,6 +47,7 @@ public class ExecuteWorkflow {
|
|||
static int numberOfPiwikIdsToDownload;
|
||||
static int numberOfSiteIdsToDownload;
|
||||
|
||||
static boolean laReferenciaEmptyDirs;
|
||||
static boolean downloadLaReferenciaLogs;
|
||||
static boolean processLaReferenciaLogs;
|
||||
|
||||
|
@ -98,6 +101,11 @@ public class ExecuteWorkflow {
|
|||
else
|
||||
recreateDbAndTables = false;
|
||||
|
||||
if (parser.get("piwikEmptyDirs").toLowerCase().equals("true"))
|
||||
piwikEmptyDirs = true;
|
||||
else
|
||||
piwikEmptyDirs = false;
|
||||
|
||||
if (parser.get("downloadPiwikLogs").toLowerCase().equals("true"))
|
||||
downloadPiwikLogs = true;
|
||||
else
|
||||
|
@ -119,6 +127,11 @@ public class ExecuteWorkflow {
|
|||
numberOfPiwikIdsToDownload = Integer.parseInt(parser.get("numberOfPiwikIdsToDownload"));
|
||||
numberOfSiteIdsToDownload = Integer.parseInt(parser.get("numberOfSiteIdsToDownload"));
|
||||
|
||||
if (parser.get("laReferenciaEmptyDirs").toLowerCase().equals("true"))
|
||||
laReferenciaEmptyDirs = true;
|
||||
else
|
||||
laReferenciaEmptyDirs = false;
|
||||
|
||||
if (parser.get("downloadLaReferenciaLogs").toLowerCase().equals("true"))
|
||||
downloadLaReferenciaLogs = true;
|
||||
else
|
||||
|
|
|
@ -39,6 +39,22 @@ public class PiwikStatsDB {
|
|||
|
||||
}
|
||||
|
||||
public void reCreateLogDirs() throws IllegalArgumentException, IOException {
|
||||
FileSystem dfs = FileSystem.get(new Configuration());
|
||||
|
||||
logger.info("Deleting repoLog directory: " + ExecuteWorkflow.repoLogPath);
|
||||
dfs.delete(new Path(ExecuteWorkflow.repoLogPath), true);
|
||||
|
||||
logger.info("Deleting portalLog directory: " + ExecuteWorkflow.portalLogPath);
|
||||
dfs.delete(new Path(ExecuteWorkflow.portalLogPath), true);
|
||||
|
||||
logger.info("Creating repoLog directory: " + ExecuteWorkflow.repoLogPath);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.repoLogPath));
|
||||
|
||||
logger.info("Creating portalLog directory: " + ExecuteWorkflow.portalLogPath);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.portalLogPath));
|
||||
}
|
||||
|
||||
public void recreateDBAndTables() throws Exception {
|
||||
this.createDatabase();
|
||||
this.createTables();
|
||||
|
|
|
@ -64,11 +64,13 @@ public class UsageStatsExporter {
|
|||
logger.info("Initializing the download logs module");
|
||||
PiwikDownloadLogs piwd = new PiwikDownloadLogs(ExecuteWorkflow.matomoBaseURL, ExecuteWorkflow.matomoAuthToken);
|
||||
|
||||
if (ExecuteWorkflow.piwikEmptyDirs) {
|
||||
logger.info("Recreating Piwik log directories");
|
||||
piwikstatsdb.reCreateLogDirs();
|
||||
}
|
||||
|
||||
// Downloading piwik logs (also managing directory creation)
|
||||
if (ExecuteWorkflow.downloadPiwikLogs) {
|
||||
logger.info("Recreating log directories");
|
||||
reCreateLogDirs();
|
||||
|
||||
logger.info("Downloading piwik logs");
|
||||
piwd
|
||||
.GetOpenAIRELogs(
|
||||
|
@ -90,6 +92,11 @@ public class UsageStatsExporter {
|
|||
LaReferenciaDownloadLogs lrf = new LaReferenciaDownloadLogs(ExecuteWorkflow.lareferenciaBaseURL,
|
||||
ExecuteWorkflow.lareferenciaAuthToken);
|
||||
|
||||
if (ExecuteWorkflow.laReferenciaEmptyDirs) {
|
||||
logger.info("Recreating LaReferencia log directories");
|
||||
piwikstatsdb.reCreateLogDirs();
|
||||
}
|
||||
|
||||
if (ExecuteWorkflow.downloadLaReferenciaLogs) {
|
||||
logger.info("Downloading LaReferencia logs");
|
||||
lrf.GetLaReferenciaRepos(ExecuteWorkflow.lareferenciaLogPath);
|
||||
|
|
|
@ -101,6 +101,12 @@
|
|||
"paramDescription": "Re-create database and initial tables?",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "pwed",
|
||||
"paramLongName": "piwikEmptyDirs",
|
||||
"paramDescription": "Empty piwik directories?",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "ppwl",
|
||||
"paramLongName": "processPiwikLogs",
|
||||
|
@ -137,6 +143,12 @@
|
|||
"paramDescription": "Limit the number of the downloaded siteids (La Referencia logs) to the first numberOfSiteIdsToDownload",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "lerd",
|
||||
"paramLongName": "laReferenciaEmptyDirs",
|
||||
"paramDescription": "Empty LaReferencia directories?",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "plrl",
|
||||
"paramLongName": "processLaReferenciaLogs",
|
||||
|
|
|
@ -59,12 +59,14 @@
|
|||
<arg>--usageStatsDBSchema</arg><arg>${usageStatsDBSchema}</arg>
|
||||
<arg>--statsDBSchema</arg><arg>${statsDBSchema}</arg>
|
||||
<arg>--recreateDbAndTables</arg><arg>${recreateDbAndTables}</arg>
|
||||
<arg>--piwikEmptyDirs</arg><arg>${piwikEmptyDirs}</arg>
|
||||
<arg>--downloadPiwikLogs</arg><arg>${downloadPiwikLogs}</arg>
|
||||
<arg>--processPiwikLogs</arg><arg>${processPiwikLogs}</arg>
|
||||
<arg>--startingLogPeriod</arg><arg>${startingLogPeriod}</arg>
|
||||
<arg>--endingLogPeriod</arg><arg>${endingLogPeriod}</arg>
|
||||
<arg>--numberOfPiwikIdsToDownload</arg><arg>${numberOfPiwikIdsToDownload}</arg>
|
||||
<arg>--numberOfSiteIdsToDownload</arg><arg>${numberOfSiteIdsToDownload}</arg>
|
||||
<arg>--laReferenciaEmptyDirs</arg><arg>${laReferenciaEmptyDirs}</arg>
|
||||
<arg>--downloadLaReferenciaLogs</arg><arg>${downloadLaReferenciaLogs}</arg>
|
||||
<arg>--processLaReferenciaLogs</arg><arg>${processLaReferenciaLogs}</arg>
|
||||
<arg>--irusCreateTablesEmptyDirs</arg><arg>${irusCreateTablesEmptyDirs}</arg>
|
||||
|
|
Loading…
Reference in New Issue