diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaDownloadLogs.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaDownloadLogs.java index e670f2588f..4ee2a94cdb 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaDownloadLogs.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaDownloadLogs.java @@ -115,9 +115,10 @@ public class LaReferenciaDownloadLogs { String inputLine; while ((inputLine = in.readLine()) != null) { response.append(inputLine); - response.append("\n"); +// response.append("\n"); } } + return response.toString(); } catch (Exception e) { log.error("Failed to get URL: " + e); @@ -144,7 +145,7 @@ public class LaReferenciaDownloadLogs { int laReferencialMatomoID) throws Exception { System.out.println("====> Downloading logs for LaReferencia repoid " + laReferencialMatomoID); - + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM"); Calendar start = Calendar.getInstance(); @@ -177,28 +178,28 @@ public class LaReferenciaDownloadLogs { .info( "Downloading logs for LaReferencia repoid " + laReferencialMatomoID + " and for " + sdf.format(date)); - System.out.println("====> Downloading logs for LaReferencia repoid " + laReferencialMatomoID + " and for " - + sdf.format(date)); - + System.out + .println( + "====> Downloading logs for LaReferencia repoid " + laReferencialMatomoID + " and for " + + sdf.format(date)); + String period = "&period=day&date=" + sdf.format(date); String outFolder = ""; outFolder = repoLogsPath; FileSystem fs = FileSystem.get(new Configuration()); + FSDataOutputStream fin = fs + .create( + new Path(outFolder + "/" + laReferencialMatomoID + "_LaRefPiwiklog" + sdf.format((date)) + ".json"), + true); String baseApiUrl = getPiwikLogUrl() + APImethod + "&idSite=" + laReferencialMatomoID + period + format + "&expanded=5&filter_limit=1000&token_auth=" + tokenAuth; String content = ""; - int i = 0; - while (!content.equals("[]\n")) { - - FSDataOutputStream fin = fs - .create( - new Path(outFolder + "/" + laReferencialMatomoID + "_LaRefPiwiklog" + sdf.format((date)) + "_" - + i + ".json"), - true); + JSONParser parser = new JSONParser(); + while (!content.equals("[]")) { String apiUrl = baseApiUrl; if (i > 0) { @@ -207,16 +208,22 @@ public class LaReferenciaDownloadLogs { content = getJson(apiUrl); - fin.write(content.getBytes()); + JSONArray jsonArray = (JSONArray) parser.parse(content); + for (Object aJsonArray : jsonArray) { + JSONObject jsonObjectRaw = (JSONObject) aJsonArray; + fin.write(jsonObjectRaw.toJSONString().getBytes()); + fin.writeChar('\n'); + } + System.out + .println( + "====> Downloaded part " + i + " of logs for LaReferencia repoid " + laReferencialMatomoID + + " and for " + + sdf.format(date)); i++; - fin.close(); } - // fin.close(); - // out.close(); + fin.close(); } - - // } } } diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java index 770cb11dca..1cf0b6e91e 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/LaReferenciaStats.java @@ -103,17 +103,18 @@ public class LaReferenciaStats { public void processLogs() throws Exception { try { - + System.out.println("====> Processing LaReferencia repository logs"); processlaReferenciaLog(); + System.out.println("====> LaReferencia repository logs process done"); log.info("LaReferencia repository process done"); - removeDoubleClicks(); - log.info("LaReferencia removing double clicks done"); - viewsStats(); - log.info("LaReferencia views done"); - downloadsStats(); - log.info("LaReferencia downloads done"); - updateProdTables(); - log.info("LaReferencia update productions tables done"); +// removeDoubleClicks(); +// log.info("LaReferencia removing double clicks done"); +// viewsStats(); +// log.info("LaReferencia views done"); +// downloadsStats(); +// log.info("LaReferencia downloads done"); +// updateProdTables(); +// log.info("LaReferencia update productions tables done"); } catch (Exception e) { log.error("Failed to process logs: " + e); diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java index 5ef38f9620..c849eccc9b 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java @@ -62,8 +62,9 @@ public class UsageStatsExporter { System.out.println("====> Downloading LaReferencia logs"); lrf.GetLaReferenciaRepos(lareferenciaLogPath); System.out.println("====> Downloaded LaReferencia logs"); - LaReferenciaStats lastats = new LaReferenciaStats(lareferenciaLogPath); -// lastats.processLogs(); + LaReferenciaStats lastats = new LaReferenciaStats(lareferenciaLogPath); + System.out.println("====> Processing LaReferencia logs"); + lastats.processLogs(); // log.info("LaReferencia logs done"); // IrusStats irusstats = new IrusStats(irusUKBaseURL);