diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
index 6b18140a7..a4891d788 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java
@@ -53,6 +53,11 @@ public class ExecuteWorkflow {
static boolean irusProcessStats;
static int irusNumberOfOpendoarsToDownload;
+ static boolean sarcCreateTablesEmptyDirs;
+ static boolean sarcDownloadReports;
+ static boolean sarcProcessStats;
+ static int sarcNumberOfOpendoarsToDownload;
+
public static void main(String args[]) throws Exception {
// Sending the logs to the console
@@ -135,6 +140,21 @@ public class ExecuteWorkflow {
irusProcessStats = false;
irusNumberOfOpendoarsToDownload = Integer.parseInt(parser.get("irusNumberOfOpendoarsToDownload"));
+
+ if (parser.get("sarcCreateTablesEmptyDirs").toLowerCase().equals("true"))
+ sarcCreateTablesEmptyDirs = true;
+ else
+ sarcCreateTablesEmptyDirs = false;
+ if (parser.get("sarcDownloadReports").toLowerCase().equals("true"))
+ sarcDownloadReports = true;
+ else
+ sarcDownloadReports = false;
+ if (parser.get("sarcProcessStats").toLowerCase().equals("true"))
+ sarcProcessStats = true;
+ else
+ sarcProcessStats = false;
+ sarcNumberOfOpendoarsToDownload = Integer.parseInt(parser.get("sarcNumberOfOpendoarsToDownload"));
+
UsageStatsExporter usagestatsExport = new UsageStatsExporter();
usagestatsExport.export();
}
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java
index 57fe3fcac..c9c224a55 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java
@@ -69,6 +69,22 @@ public class SarcStats {
}
}
+ public void reCreateLogDirs() throws IOException {
+ FileSystem dfs = FileSystem.get(new Configuration());
+
+ logger.info("Deleting sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
+ dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathArray), true);
+
+ logger.info("Deleting sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
+ dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathNonArray), true);
+
+ logger.info("Creating sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
+ dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathArray));
+
+ logger.info("Creating sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
+ dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathNonArray));
+ }
+
public void processSarc(String sarcsReportPathArray, String sarcsReportPathNonArray,
String url, String issn) throws Exception {
Statement stmt = ConnectDB.getHiveConnection().createStatement();
@@ -353,8 +369,6 @@ public class SarcStats {
* ".json")); wr.print(text); wr.close();
*/
- System.out.println("AAAAAAAAAAA text " + text);
-
JSONParser parser = new JSONParser();
JSONObject jsonObject = null;
try {
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
index 4318f2cde..4d66b832e 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java
@@ -60,12 +60,6 @@ public class UsageStatsExporter {
logger.info("Deleting portalLog directory: " + ExecuteWorkflow.portalLogPath);
dfs.delete(new Path(ExecuteWorkflow.portalLogPath), true);
- logger.info("Deleting sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
- dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathArray), true);
-
- logger.info("Deleting sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
- dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathNonArray), true);
-
logger.info("Deleting lareferenciaLog directory: " + ExecuteWorkflow.lareferenciaLogPath);
dfs.delete(new Path(ExecuteWorkflow.lareferenciaLogPath), true);
@@ -75,12 +69,6 @@ public class UsageStatsExporter {
logger.info("Creating portalLog directory: " + ExecuteWorkflow.portalLogPath);
dfs.mkdirs(new Path(ExecuteWorkflow.portalLogPath));
- logger.info("Creating sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
- dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathArray));
-
- logger.info("Creating sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
- dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathNonArray));
-
logger.info("Creating lareferenciaLog directory: " + ExecuteWorkflow.lareferenciaLogPath);
dfs.mkdirs(new Path(ExecuteWorkflow.lareferenciaLogPath));
}
@@ -161,11 +149,18 @@ public class UsageStatsExporter {
}
System.exit(0);
-
-// SarcStats sarcStats = new SarcStats();
-// sarcStats.getAndProcessSarc(sarcsReportPathArray, sarcsReportPathNonArray);
-// sarcStats.finalizeSarcStats();
-// log.info("sarc done");
+
+ SarcStats sarcStats = new SarcStats();
+ if (ExecuteWorkflow.sarcCreateTablesEmptyDirs) {
+ sarcStats.reCreateLogDirs();
+ }
+ if (ExecuteWorkflow.sarcDownloadReports) {
+ sarcStats.getAndProcessSarc(ExecuteWorkflow.sarcsReportPathArray, ExecuteWorkflow.sarcsReportPathNonArray);
+ }
+ if (ExecuteWorkflow.sarcProcessStats) {
+ sarcStats.finalizeSarcStats();
+ }
+ logger.info("Sarc done");
// // finalize usagestats
// piwikstatsdb.finalizeStats();
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
index 48cabfde7..cb31744cb 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json
@@ -172,5 +172,29 @@
"paramLongName": "irusNumberOfOpendoarsToDownload",
"paramDescription": "Limit the number of the downloaded Opendoars (Irus) to the first irusNumberOfOpendoarsToDownload",
"paramRequired": true
+ },
+ {
+ "paramName": "icted",
+ "paramLongName": "sarcCreateTablesEmptyDirs",
+ "paramDescription": "Sarc section: Create tables and empty JSON directories?",
+ "paramRequired": true
+ },
+ {
+ "paramName": "idr",
+ "paramLongName": "sarcDownloadReports",
+ "paramDescription": "Sarc section: Download reports?",
+ "paramRequired": true
+ },
+ {
+ "paramName": "ipr",
+ "paramLongName": "sarcProcessStats",
+ "paramDescription": "Sarc section: Process stats?",
+ "paramRequired": true
+ },
+ {
+ "paramName": "inod",
+ "paramLongName": "sarcNumberOfOpendoarsToDownload",
+ "paramDescription": "Limit the number of the downloaded Opendoars (Sarc) to the first sarcNumberOfOpendoarsToDownload",
+ "paramRequired": true
}
]
diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
index b14e9408f..a900d8f49 100644
--- a/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
+++ b/dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml
@@ -71,6 +71,10 @@
--irusDownloadReports${irusDownloadReports}
--irusProcessStats${irusProcessStats}
--irusNumberOfOpendoarsToDownload${irusNumberOfOpendoarsToDownload}
+ --sarcCreateTablesEmptyDirs${sarcCreateTablesEmptyDirs}
+ --sarcDownloadReports${sarcDownloadReports}
+ --sarcProcessStats${sarcProcessStats}
+ --sarcNumberOfOpendoarsToDownload${sarcNumberOfOpendoarsToDownload}