Better organisation of downloaded logs

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaDownloadLogs.java

@@ -115,9 +115,10 @@ public class LaReferenciaDownloadLogs {
 				String inputLine;
 				while ((inputLine = in.readLine()) != null) {
 					response.append(inputLine);
-					response.append("\n");
+//					response.append("\n");
 				}
 			}
+
 			return response.toString();
 		} catch (Exception e) {
 			log.error("Failed to get URL: " + e);
@@ -177,28 +178,28 @@ public class LaReferenciaDownloadLogs {
 				.info(
 					"Downloading logs for LaReferencia repoid " + laReferencialMatomoID + " and for "
 						+ sdf.format(date));
-			System.out.println("====> Downloading logs for LaReferencia repoid " + laReferencialMatomoID + " and for "
-					+ sdf.format(date));
+			System.out
+				.println(
+					"====> Downloading logs for LaReferencia repoid " + laReferencialMatomoID + " and for "
+						+ sdf.format(date));
 
 			String period = "&period=day&date=" + sdf.format(date);
 			String outFolder = "";
 			outFolder = repoLogsPath;
 
 			FileSystem fs = FileSystem.get(new Configuration());
+			FSDataOutputStream fin = fs
+				.create(
+					new Path(outFolder + "/" + laReferencialMatomoID + "_LaRefPiwiklog" + sdf.format((date)) + ".json"),
+					true);
 
 			String baseApiUrl = getPiwikLogUrl() + APImethod + "&idSite=" + laReferencialMatomoID + period + format
 				+ "&expanded=5&filter_limit=1000&token_auth=" + tokenAuth;
 			String content = "";
-
 			int i = 0;
 
-			while (!content.equals("[]\n")) {
-
-				FSDataOutputStream fin = fs
-					.create(
-						new Path(outFolder + "/" + laReferencialMatomoID + "_LaRefPiwiklog" + sdf.format((date)) + "_"
-							+ i + ".json"),
-						true);
+			JSONParser parser = new JSONParser();
+			while (!content.equals("[]")) {
 				String apiUrl = baseApiUrl;
 
 				if (i > 0) {
@@ -207,16 +208,22 @@ public class LaReferenciaDownloadLogs {
 
 				content = getJson(apiUrl);
 
-				fin.write(content.getBytes());
+				JSONArray jsonArray = (JSONArray) parser.parse(content);
+				for (Object aJsonArray : jsonArray) {
+					JSONObject jsonObjectRaw = (JSONObject) aJsonArray;
+					fin.write(jsonObjectRaw.toJSONString().getBytes());
+					fin.writeChar('\n');
+				}
 
+				System.out
+					.println(
+						"====> Downloaded part " + i + " of logs for LaReferencia repoid " + laReferencialMatomoID
+							+ " and for "
+							+ sdf.format(date));
 				i++;
-				fin.close();
 			}
-			// fin.close();
-			// out.close();
+			fin.close();
 
 		}
-
-		// }
 	}
 }

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java

@@ -103,17 +103,18 @@ public class LaReferenciaStats {
 
 	public void processLogs() throws Exception {
 		try {
-
+			System.out.println("====> Processing LaReferencia repository logs");
 			processlaReferenciaLog();
+			System.out.println("====> LaReferencia repository logs process done");
 			log.info("LaReferencia repository process done");
-			removeDoubleClicks();
-			log.info("LaReferencia removing double clicks done");
-			viewsStats();
-			log.info("LaReferencia views done");
-			downloadsStats();
-			log.info("LaReferencia downloads done");
-			updateProdTables();
-			log.info("LaReferencia update productions tables done");
+//			removeDoubleClicks();
+//			log.info("LaReferencia removing double clicks done");
+//			viewsStats();
+//			log.info("LaReferencia views done");
+//			downloadsStats();
+//			log.info("LaReferencia downloads done");
+//			updateProdTables();
+//			log.info("LaReferencia update productions tables done");
 
 		} catch (Exception e) {
 			log.error("Failed to process logs: " + e);

dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java

@@ -62,8 +62,9 @@ public class UsageStatsExporter {
 		System.out.println("====> Downloading LaReferencia logs");
 		lrf.GetLaReferenciaRepos(lareferenciaLogPath);
 		System.out.println("====> Downloaded LaReferencia logs");
-        LaReferenciaStats lastats = new LaReferenciaStats(lareferenciaLogPath);
-//        lastats.processLogs();
+		LaReferenciaStats lastats = new LaReferenciaStats(lareferenciaLogPath);
+		System.out.println("====> Processing LaReferencia logs");
+		lastats.processLogs();
 //        log.info("LaReferencia logs done");
 
 //		IrusStats irusstats = new IrusStats(irusUKBaseURL);