forked from D-Net/dnet-hadoop
Flags & limiters for Sarc stats
This commit is contained in:
parent
2de17e7f32
commit
715bbd487d
|
|
@ -53,6 +53,11 @@ public class ExecuteWorkflow {
|
|||
static boolean irusProcessStats;
|
||||
static int irusNumberOfOpendoarsToDownload;
|
||||
|
||||
static boolean sarcCreateTablesEmptyDirs;
|
||||
static boolean sarcDownloadReports;
|
||||
static boolean sarcProcessStats;
|
||||
static int sarcNumberOfOpendoarsToDownload;
|
||||
|
||||
public static void main(String args[]) throws Exception {
|
||||
|
||||
// Sending the logs to the console
|
||||
|
|
@ -135,6 +140,21 @@ public class ExecuteWorkflow {
|
|||
irusProcessStats = false;
|
||||
irusNumberOfOpendoarsToDownload = Integer.parseInt(parser.get("irusNumberOfOpendoarsToDownload"));
|
||||
|
||||
|
||||
if (parser.get("sarcCreateTablesEmptyDirs").toLowerCase().equals("true"))
|
||||
sarcCreateTablesEmptyDirs = true;
|
||||
else
|
||||
sarcCreateTablesEmptyDirs = false;
|
||||
if (parser.get("sarcDownloadReports").toLowerCase().equals("true"))
|
||||
sarcDownloadReports = true;
|
||||
else
|
||||
sarcDownloadReports = false;
|
||||
if (parser.get("sarcProcessStats").toLowerCase().equals("true"))
|
||||
sarcProcessStats = true;
|
||||
else
|
||||
sarcProcessStats = false;
|
||||
sarcNumberOfOpendoarsToDownload = Integer.parseInt(parser.get("sarcNumberOfOpendoarsToDownload"));
|
||||
|
||||
UsageStatsExporter usagestatsExport = new UsageStatsExporter();
|
||||
usagestatsExport.export();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,22 @@ public class SarcStats {
|
|||
}
|
||||
}
|
||||
|
||||
public void reCreateLogDirs() throws IOException {
|
||||
FileSystem dfs = FileSystem.get(new Configuration());
|
||||
|
||||
logger.info("Deleting sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
|
||||
dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathArray), true);
|
||||
|
||||
logger.info("Deleting sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
|
||||
dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathNonArray), true);
|
||||
|
||||
logger.info("Creating sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathArray));
|
||||
|
||||
logger.info("Creating sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathNonArray));
|
||||
}
|
||||
|
||||
public void processSarc(String sarcsReportPathArray, String sarcsReportPathNonArray,
|
||||
String url, String issn) throws Exception {
|
||||
Statement stmt = ConnectDB.getHiveConnection().createStatement();
|
||||
|
|
@ -353,8 +369,6 @@ public class SarcStats {
|
|||
* ".json")); wr.print(text); wr.close();
|
||||
*/
|
||||
|
||||
System.out.println("AAAAAAAAAAA text " + text);
|
||||
|
||||
JSONParser parser = new JSONParser();
|
||||
JSONObject jsonObject = null;
|
||||
try {
|
||||
|
|
|
|||
|
|
@ -60,12 +60,6 @@ public class UsageStatsExporter {
|
|||
logger.info("Deleting portalLog directory: " + ExecuteWorkflow.portalLogPath);
|
||||
dfs.delete(new Path(ExecuteWorkflow.portalLogPath), true);
|
||||
|
||||
logger.info("Deleting sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
|
||||
dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathArray), true);
|
||||
|
||||
logger.info("Deleting sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
|
||||
dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathNonArray), true);
|
||||
|
||||
logger.info("Deleting lareferenciaLog directory: " + ExecuteWorkflow.lareferenciaLogPath);
|
||||
dfs.delete(new Path(ExecuteWorkflow.lareferenciaLogPath), true);
|
||||
|
||||
|
|
@ -75,12 +69,6 @@ public class UsageStatsExporter {
|
|||
logger.info("Creating portalLog directory: " + ExecuteWorkflow.portalLogPath);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.portalLogPath));
|
||||
|
||||
logger.info("Creating sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathArray));
|
||||
|
||||
logger.info("Creating sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathNonArray));
|
||||
|
||||
logger.info("Creating lareferenciaLog directory: " + ExecuteWorkflow.lareferenciaLogPath);
|
||||
dfs.mkdirs(new Path(ExecuteWorkflow.lareferenciaLogPath));
|
||||
}
|
||||
|
|
@ -162,10 +150,17 @@ public class UsageStatsExporter {
|
|||
|
||||
System.exit(0);
|
||||
|
||||
// SarcStats sarcStats = new SarcStats();
|
||||
// sarcStats.getAndProcessSarc(sarcsReportPathArray, sarcsReportPathNonArray);
|
||||
// sarcStats.finalizeSarcStats();
|
||||
// log.info("sarc done");
|
||||
SarcStats sarcStats = new SarcStats();
|
||||
if (ExecuteWorkflow.sarcCreateTablesEmptyDirs) {
|
||||
sarcStats.reCreateLogDirs();
|
||||
}
|
||||
if (ExecuteWorkflow.sarcDownloadReports) {
|
||||
sarcStats.getAndProcessSarc(ExecuteWorkflow.sarcsReportPathArray, ExecuteWorkflow.sarcsReportPathNonArray);
|
||||
}
|
||||
if (ExecuteWorkflow.sarcProcessStats) {
|
||||
sarcStats.finalizeSarcStats();
|
||||
}
|
||||
logger.info("Sarc done");
|
||||
|
||||
// // finalize usagestats
|
||||
// piwikstatsdb.finalizeStats();
|
||||
|
|
|
|||
|
|
@ -172,5 +172,29 @@
|
|||
"paramLongName": "irusNumberOfOpendoarsToDownload",
|
||||
"paramDescription": "Limit the number of the downloaded Opendoars (Irus) to the first irusNumberOfOpendoarsToDownload",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "icted",
|
||||
"paramLongName": "sarcCreateTablesEmptyDirs",
|
||||
"paramDescription": "Sarc section: Create tables and empty JSON directories?",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "idr",
|
||||
"paramLongName": "sarcDownloadReports",
|
||||
"paramDescription": "Sarc section: Download reports?",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "ipr",
|
||||
"paramLongName": "sarcProcessStats",
|
||||
"paramDescription": "Sarc section: Process stats?",
|
||||
"paramRequired": true
|
||||
},
|
||||
{
|
||||
"paramName": "inod",
|
||||
"paramLongName": "sarcNumberOfOpendoarsToDownload",
|
||||
"paramDescription": "Limit the number of the downloaded Opendoars (Sarc) to the first sarcNumberOfOpendoarsToDownload",
|
||||
"paramRequired": true
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -71,6 +71,10 @@
|
|||
<arg>--irusDownloadReports</arg><arg>${irusDownloadReports}</arg>
|
||||
<arg>--irusProcessStats</arg><arg>${irusProcessStats}</arg>
|
||||
<arg>--irusNumberOfOpendoarsToDownload</arg><arg>${irusNumberOfOpendoarsToDownload}</arg>
|
||||
<arg>--sarcCreateTablesEmptyDirs</arg><arg>${sarcCreateTablesEmptyDirs}</arg>
|
||||
<arg>--sarcDownloadReports</arg><arg>${sarcDownloadReports}</arg>
|
||||
<arg>--sarcProcessStats</arg><arg>${sarcProcessStats}</arg>
|
||||
<arg>--sarcNumberOfOpendoarsToDownload</arg><arg>${sarcNumberOfOpendoarsToDownload}</arg>
|
||||
<capture-output/>
|
||||
</java>
|
||||
<ok to="End" />
|
||||
|
|
|
|||
Loading…
Reference in New Issue