Flags for La Referencia logs

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/ExecuteWorkflow.java

@@ -37,12 +37,16 @@ public class ExecuteWorkflow {
 	static String usageStatsDBSchema;
 	static String statsDBSchema;
 	static boolean recreateDbAndTables;
-	static boolean downloadLogs;
+	static boolean downloadPiwikLogs;
+	static boolean processPiwikLogs;
+
 	static Calendar startingLogPeriod;
 	static Calendar endingLogPeriod;
 	static int numberOfPiwikIdsToDownload;
 	static int numberOfSiteIdsToDownload;
-	static boolean processPiwikLogs;
+
+	static boolean downloadLaReferenciaLogs;
+	static boolean processLaReferenciaLogs;
 
 	public static void main(String args[]) throws Exception {
 
@@ -81,10 +85,10 @@ public class ExecuteWorkflow {
 		else
 			recreateDbAndTables = false;
 
-		if (parser.get("downloadLogs").toLowerCase().equals("true"))
-			downloadLogs = true;
+		if (parser.get("downloadPiwikLogs").toLowerCase().equals("true"))
+			downloadPiwikLogs = true;
 		else
-			downloadLogs = false;
+			downloadPiwikLogs = false;
 
 		if (parser.get("processPiwikLogs").toLowerCase().equals("true"))
 			processPiwikLogs = true;
@@ -102,6 +106,16 @@ public class ExecuteWorkflow {
 		numberOfPiwikIdsToDownload = Integer.parseInt(parser.get("numberOfPiwikIdsToDownload"));
 		numberOfSiteIdsToDownload = Integer.parseInt(parser.get("numberOfSiteIdsToDownload"));
 
+		if (parser.get("downloadLaReferenciaLogs").toLowerCase().equals("true"))
+			downloadLaReferenciaLogs = true;
+		else
+			downloadLaReferenciaLogs = false;
+
+		if (parser.get("processLaReferenciaLogs").toLowerCase().equals("true"))
+			processLaReferenciaLogs = true;
+		else
+			processLaReferenciaLogs = false;
+
 		UsageStatsExporter usagestatsExport = new UsageStatsExporter();
 		usagestatsExport.export();
 	}

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java

@@ -109,18 +109,18 @@ public class LaReferenciaStats {
 	public void processLogs() throws Exception {
 		try {
 			logger.info("Processing LaReferencia repository logs");
-//			processlaReferenciaLog();
+			processlaReferenciaLog();
 			logger.info("LaReferencia repository logs process done");
 
 			logger.info("LaReferencia removing double clicks");
-//			removeDoubleClicks();
+			removeDoubleClicks();
 			logger.info("LaReferencia removed double clicks");
 
 			logger.info("LaReferencia creating viewsStats");
-//			viewsStats();
+			viewsStats();
 			logger.info("LaReferencia created viewsStats");
 			logger.info("LaReferencia creating downloadsStats");
-//			downloadsStats();
+			downloadsStats();
 			logger.info("LaReferencia created downloadsStats");
 			logger.info("LaReferencia updating Production Tables");
 			updateProdTables();

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java

@@ -109,7 +109,7 @@ public class UsageStatsExporter {
 		PiwikDownloadLogs piwd = new PiwikDownloadLogs(ExecuteWorkflow.matomoBaseURL, ExecuteWorkflow.matomoAuthToken);
 
 		// Downloading piwik logs (also managing directory creation)
-		if (ExecuteWorkflow.downloadLogs) {
+		if (ExecuteWorkflow.downloadPiwikLogs) {
 			logger.info("Recreating log directories");
 			reCreateLogDirs();
 
@@ -133,14 +133,19 @@ public class UsageStatsExporter {
 		logger.info("Creating LaReferencia tables");
 		LaReferenciaDownloadLogs lrf = new LaReferenciaDownloadLogs(ExecuteWorkflow.lareferenciaBaseURL,
 			ExecuteWorkflow.lareferenciaAuthToken);
-		logger.info("Downloading LaReferencia logs");
-		lrf.GetLaReferenciaRepos(ExecuteWorkflow.lareferenciaLogPath);
-		logger.info("Downloaded LaReferencia logs");
-		LaReferenciaStats lastats = new LaReferenciaStats(ExecuteWorkflow.lareferenciaLogPath);
-		logger.info("Processing LaReferencia logs");
-//		lastats.processLogs();
-//        log.info("LaReferencia logs done");
 
+		if (ExecuteWorkflow.downloadLaReferenciaLogs) {
+			logger.info("Downloading LaReferencia logs");
+			lrf.GetLaReferenciaRepos(ExecuteWorkflow.lareferenciaLogPath);
+			logger.info("Downloaded LaReferencia logs");
+		}
+		LaReferenciaStats lastats = new LaReferenciaStats(ExecuteWorkflow.lareferenciaLogPath);
+
+		if (ExecuteWorkflow.processLaReferenciaLogs) {
+			logger.info("Processing LaReferencia logs");
+			lastats.processLogs();
+			logger.info("LaReferencia logs done");
+		}
 		System.exit(0);
 
 //		IrusStats irusstats = new IrusStats(irusUKBaseURL);
@@ -155,7 +160,7 @@ public class UsageStatsExporter {
 //		log.info("sarc done");
 
 //		// finalize usagestats
-		piwikstatsdb.finalizeStats();
+//		piwikstatsdb.finalizeStats();
 //		log.info("finalized stats");
 	}
 }

dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/export/usagestats_parameters.json

@@ -108,9 +108,9 @@
     "paramRequired": true
   },
   {
-    "paramName": "dl",
-    "paramLongName": "downloadLogs",
-    "paramDescription": "download logs?",
+    "paramName": "dpwl",
+    "paramLongName": "downloadPiwikLogs",
+    "paramDescription": "download piwik logs?",
     "paramRequired": true
   },
   {
@@ -136,6 +136,18 @@
     "paramLongName": "numberOfSiteIdsToDownload",
     "paramDescription": "Limit the number of the downloaded siteids (La Referencia logs) to the first numberOfSiteIdsToDownload",
     "paramRequired": true
+  },
+  {
+    "paramName": "plrl",
+    "paramLongName": "processLaReferenciaLogs",
+    "paramDescription": "Process the La Referencia logs (create & fill in the needed tables and process the data) based on the downloaded data",
+    "paramRequired": true
+  },
+  {
+    "paramName": "dlrl",
+    "paramLongName": "downloadLaReferenciaLogs",
+    "paramDescription": "download La Referencia logs?",
+    "paramRequired": true
   }
 ]
 

dhp-workflows/dhp-usage-stats-update/src/main/resources/eu/dnetlib/dhp/oa/graph/usagestats/oozie_app/workflow.xml

@@ -59,12 +59,14 @@
             <arg>--usageStatsDBSchema</arg><arg>${usageStatsDBSchema}</arg>
             <arg>--statsDBSchema</arg><arg>${statsDBSchema}</arg>
             <arg>--recreateDbAndTables</arg><arg>${recreateDbAndTables}</arg>
-            <arg>--downloadLogs</arg><arg>${downloadLogs}</arg>
+            <arg>--downloadPiwikLogs</arg><arg>${downloadPiwikLogs}</arg>
+            <arg>--processPiwikLogs</arg><arg>${processPiwikLogs}</arg>
             <arg>--startingLogPeriod</arg><arg>${startingLogPeriod}</arg>
             <arg>--endingLogPeriod</arg><arg>${endingLogPeriod}</arg>
             <arg>--numberOfPiwikIdsToDownload</arg><arg>${numberOfPiwikIdsToDownload}</arg>
             <arg>--numberOfSiteIdsToDownload</arg><arg>${numberOfSiteIdsToDownload}</arg>
-            <arg>--processPiwikLogs</arg><arg>${processPiwikLogs}</arg>
+            <arg>--downloadLaReferenciaLogs</arg><arg>${downloadLaReferenciaLogs}</arg>
+            <arg>--processLaReferenciaLogs</arg><arg>${processLaReferenciaLogs}</arg>
             <capture-output/>
         </java>
         <ok to="End" />