Flags & limiters for Sarc stats

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java

@@ -53,6 +53,11 @@ public class ExecuteWorkflow {
 	static boolean irusProcessStats;
 	static int irusNumberOfOpendoarsToDownload;
 
+	static boolean sarcCreateTablesEmptyDirs;
+	static boolean sarcDownloadReports;
+	static boolean sarcProcessStats;
+	static int sarcNumberOfOpendoarsToDownload;
+	
 	public static void main(String args[]) throws Exception {
 
 		// Sending the logs to the console
@@ -135,6 +140,21 @@ public class ExecuteWorkflow {
 			irusProcessStats = false;
 		irusNumberOfOpendoarsToDownload = Integer.parseInt(parser.get("irusNumberOfOpendoarsToDownload"));
 
+		
+		if (parser.get("sarcCreateTablesEmptyDirs").toLowerCase().equals("true"))
+			sarcCreateTablesEmptyDirs = true;
+		else
+			sarcCreateTablesEmptyDirs = false;
+		if (parser.get("sarcDownloadReports").toLowerCase().equals("true"))
+			sarcDownloadReports = true;
+		else
+			sarcDownloadReports = false;
+		if (parser.get("sarcProcessStats").toLowerCase().equals("true"))
+			sarcProcessStats = true;
+		else
+			sarcProcessStats = false;
+		sarcNumberOfOpendoarsToDownload = Integer.parseInt(parser.get("sarcNumberOfOpendoarsToDownload"));
+		
 		UsageStatsExporter usagestatsExport = new UsageStatsExporter();
 		usagestatsExport.export();
 	}

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java

@@ -69,6 +69,22 @@ public class SarcStats {
 		}
 	}
 
+	public void reCreateLogDirs() throws IOException {
+		FileSystem dfs = FileSystem.get(new Configuration());
+		
+		logger.info("Deleting sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
+		dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathArray), true);
+
+		logger.info("Deleting sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
+		dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathNonArray), true);
+		
+		logger.info("Creating sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
+		dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathArray));
+
+		logger.info("Creating sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
+		dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathNonArray));
+	}
+	
 	public void processSarc(String sarcsReportPathArray, String sarcsReportPathNonArray,
 		String url, String issn) throws Exception {
 		Statement stmt = ConnectDB.getHiveConnection().createStatement();
@@ -353,8 +369,6 @@ public class SarcStats {
 			 * ".json")); wr.print(text); wr.close();
 			 */
 
-			System.out.println("AAAAAAAAAAA text " + text);
-
 			JSONParser parser = new JSONParser();
 			JSONObject jsonObject = null;
 			try {

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java

@@ -60,12 +60,6 @@ public class UsageStatsExporter {
 		logger.info("Deleting portalLog directory: " + ExecuteWorkflow.portalLogPath);
 		dfs.delete(new Path(ExecuteWorkflow.portalLogPath), true);
 
-		logger.info("Deleting sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
-		dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathArray), true);
-
-		logger.info("Deleting sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
-		dfs.delete(new Path(ExecuteWorkflow.sarcsReportPathNonArray), true);
-
 		logger.info("Deleting lareferenciaLog directory: " + ExecuteWorkflow.lareferenciaLogPath);
 		dfs.delete(new Path(ExecuteWorkflow.lareferenciaLogPath), true);
 
@@ -75,12 +69,6 @@ public class UsageStatsExporter {
 		logger.info("Creating portalLog directory: " + ExecuteWorkflow.portalLogPath);
 		dfs.mkdirs(new Path(ExecuteWorkflow.portalLogPath));
 
-		logger.info("Creating sarcsReport (Array) directory: " + ExecuteWorkflow.sarcsReportPathArray);
-		dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathArray));
-
-		logger.info("Creating sarcsReport (NonArray) directory: " + ExecuteWorkflow.sarcsReportPathNonArray);
-		dfs.mkdirs(new Path(ExecuteWorkflow.sarcsReportPathNonArray));
-
 		logger.info("Creating lareferenciaLog directory: " + ExecuteWorkflow.lareferenciaLogPath);
 		dfs.mkdirs(new Path(ExecuteWorkflow.lareferenciaLogPath));
 	}
@@ -161,11 +149,18 @@ public class UsageStatsExporter {
 		}
 
 		System.exit(0);
-
+		
-//		SarcStats sarcStats = new SarcStats();
+		SarcStats sarcStats = new SarcStats();
-//		sarcStats.getAndProcessSarc(sarcsReportPathArray, sarcsReportPathNonArray);
+		if (ExecuteWorkflow.sarcCreateTablesEmptyDirs) {
-//		sarcStats.finalizeSarcStats();
+			sarcStats.reCreateLogDirs();
-//		log.info("sarc done");
+		}
+		if (ExecuteWorkflow.sarcDownloadReports) {
+			sarcStats.getAndProcessSarc(ExecuteWorkflow.sarcsReportPathArray, ExecuteWorkflow.sarcsReportPathNonArray);
+		}
+		if (ExecuteWorkflow.sarcProcessStats) {
+			sarcStats.finalizeSarcStats();
+		}
+		logger.info("Sarc done");
 
 //		// finalize usagestats
 //		piwikstatsdb.finalizeStats();

dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json

@@ -172,5 +172,29 @@
     "paramLongName": "irusNumberOfOpendoarsToDownload",
     "paramDescription": "Limit the number of the downloaded Opendoars (Irus) to the first irusNumberOfOpendoarsToDownload",
     "paramRequired": true
+  },
+  {
+    "paramName": "icted",
+    "paramLongName": "sarcCreateTablesEmptyDirs",
+    "paramDescription": "Sarc section: Create tables and empty JSON directories?",
+    "paramRequired": true
+  },
+  {
+    "paramName": "idr",
+    "paramLongName": "sarcDownloadReports",
+    "paramDescription": "Sarc section: Download reports?",
+    "paramRequired": true
+  },
+  {
+    "paramName": "ipr",
+    "paramLongName": "sarcProcessStats",
+    "paramDescription": "Sarc section: Process stats?",
+    "paramRequired": true
+  },
+  {
+    "paramName": "inod",
+    "paramLongName": "sarcNumberOfOpendoarsToDownload",
+    "paramDescription": "Limit the number of the downloaded Opendoars (Sarc) to the first sarcNumberOfOpendoarsToDownload",
+    "paramRequired": true
   }
 ]

dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml

@@ -71,6 +71,10 @@
             <arg>--irusDownloadReports</arg><arg>${irusDownloadReports}</arg>
             <arg>--irusProcessStats</arg><arg>${irusProcessStats}</arg>
             <arg>--irusNumberOfOpendoarsToDownload</arg><arg>${irusNumberOfOpendoarsToDownload}</arg>
+            <arg>--sarcCreateTablesEmptyDirs</arg><arg>${sarcCreateTablesEmptyDirs}</arg>
+            <arg>--sarcDownloadReports</arg><arg>${sarcDownloadReports}</arg>
+            <arg>--sarcProcessStats</arg><arg>${sarcProcessStats}</arg>
+            <arg>--sarcNumberOfOpendoarsToDownload</arg><arg>${sarcNumberOfOpendoarsToDownload}</arg>
             <capture-output/>
         </java>
         <ok to="End" />