From 1dcb197f025237b39622df8ae4f10a565b9eef4d Mon Sep 17 00:00:00 2001 From: Spyros Zoupanos Date: Wed, 16 Sep 2020 21:28:05 +0300 Subject: [PATCH] Renamings on Sarcs and deletion of empty files (to be double checked) --- .../oa/graph/usagestats/export/SarcStats.java | 88 +++++++++++-------- .../usagestats/export/UsageStatsExporter.java | 2 +- 2 files changed, 52 insertions(+), 38 deletions(-) diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java index db7828a30a..5245a4a796 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java @@ -65,57 +65,61 @@ public class SarcStats { } } - public void processSarc(String sarcsReportPathArray, String sarcsReportPathNonArray) throws Exception { + public void processSarc() { + + } + + public void getSarc(String sarcsReportPathArray, String sarcsReportPathNonArray) throws Exception { // There was a problem to download the following file - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/", "0873-819X"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/", "1646-2335"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/cct/sushiLite/v1_7/", "2182-3030"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://actapediatrica.spp.pt/sushiLite/v1_7/", "0873-9781"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/sociologiapp/sushiLite/v1_7/", "0873-6529"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/finisterra/sushiLite/v1_7/", "0430-5027"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/sisyphus/sushiLite/v1_7/", "2182-8474"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/anestesiologia/sushiLite/v1_7/", "0871-6099"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/rpe/sushiLite/v1_7/", "0871-9187"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/psilogos/sushiLite/v1_7/", "1646-091X"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/juridica/sushiLite/v1_7/", "2183-5799"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/ecr/sushiLite/v1_7/", "1647-2098"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/nascercrescer/sushiLite/v1_7/", "0872-0754"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/cea/sushiLite/v1_7/", "1645-3794"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/proelium/sushiLite/v1_7/", "1645-8826"); - processARReport( + getARReport( sarcsReportPathArray, sarcsReportPathNonArray, "https://revistas.rcaap.pt/millenium/sushiLite/v1_7/", "0873-3015"); } @@ -136,7 +140,7 @@ public class SarcStats { ConnectDB.getConnection().close(); } - public void processARReport(String sarcsReportPathArray, String sarcsReportPathNonArray, + public void getARReport(String sarcsReportPathArray, String sarcsReportPathNonArray, String url, String issn) throws Exception { log.info("Processing SARC! issn: " + issn + " with url: " + url); ConnectDB.getConnection().setAutoCommit(false); @@ -192,6 +196,7 @@ public class SarcStats { * PrintWriter wr = new PrintWriter(new FileWriter("logs/" + simpleDateFormat.format(start.getTime()) + * ".json")); wr.print(text); wr.close(); */ + System.out.println("AAAAAAAAAAA text " + text); JSONParser parser = new JSONParser(); @@ -225,14 +230,14 @@ public class SarcStats { // Creating the file in the filesystem for the ItemIdentifier as array object FileSystem fsArray = FileSystem.get(new Configuration()); - String filePathArray = sarcsReportPathArray + "/" + "SarcsARReport_" + + String filePathArray = sarcsReportPathArray + "/" + "SarcsARReport_" + issn + "_" + simpleDateFormat.format(start.getTime()) + ".json"; System.out.println("Storing to file: " + filePathArray); FSDataOutputStream finArray = fsArray.create(new Path(filePathArray), true); // Creating the file in the filesystem for the ItemIdentifier as array object FileSystem fsNonArray = FileSystem.get(new Configuration()); - String filePathNonArray = sarcsReportPathNonArray + "/" + "SarcsARReport_" + + String filePathNonArray = sarcsReportPathNonArray + "/" + "SarcsARReport_" + issn + "_" + simpleDateFormat.format(start.getTime()) + ".json"; System.out.println("Storing to file: " + filePathNonArray); FSDataOutputStream finNonArray = fsNonArray.create(new Path(filePathNonArray), true); @@ -252,7 +257,15 @@ public class SarcStats { finArray.writeChar('\n'); } } + + if (finArray.size() == 0) + fsArray.deleteOnExit(new Path(filePathArray)); + + if (finNonArray.size() == 0) + fsNonArray.deleteOnExit(new Path(filePathNonArray)); + finArray.close(); + finNonArray.close(); ////////////////// // JSONObject jsonObjectRow = (JSONObject) aJsonArray; @@ -313,37 +326,38 @@ public class SarcStats { ConnectDB.getConnection().close(); } - + private void renameKeysRecursively(String delimiter, JSONArray givenJsonObj) throws Exception { - for(Object jjval : givenJsonObj) { - if(jjval instanceof JSONArray) - renameKeysRecursively(delimiter, (JSONArray)jjval); - else if(jjval instanceof JSONObject) - renameKeysRecursively(delimiter, (JSONObject)jjval); + for (Object jjval : givenJsonObj) { + if (jjval instanceof JSONArray) + renameKeysRecursively(delimiter, (JSONArray) jjval); + else if (jjval instanceof JSONObject) + renameKeysRecursively(delimiter, (JSONObject) jjval); // All other types of vals - else; + else + ; } } - + private void renameKeysRecursively(String delimiter, JSONObject givenJsonObj) throws Exception { Set jkeys = new HashSet(givenJsonObj.keySet()); for (String jkey : jkeys) { System.out.println("++++> " + jkey); - + String[] splitArray = jkey.split(delimiter); String newJkey = splitArray[splitArray.length - 1]; System.out.println("New jkey: " + jkey); - + Object jval = givenJsonObj.get(jkey); System.out.println("jval ===> " + jval.getClass().getName()); givenJsonObj.remove(jkey); givenJsonObj.put(newJkey, jval); - - if(jval instanceof JSONObject) - renameKeysRecursively(delimiter, (JSONObject)jval); - - if(jval instanceof JSONArray) { - renameKeysRecursively(delimiter, (JSONArray)jval); + + if (jval instanceof JSONObject) + renameKeysRecursively(delimiter, (JSONObject) jval); + + if (jval instanceof JSONArray) { + renameKeysRecursively(delimiter, (JSONArray) jval); } } } diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java index 80d8446fc1..354802a24e 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/UsageStatsExporter.java @@ -60,7 +60,7 @@ public class UsageStatsExporter { // log.info("irus done"); SarcStats sarcStats = new SarcStats(); - sarcStats.processSarc(sarcsReportPathArray, sarcsReportPathNonArray); + sarcStats.getSarc(sarcsReportPathArray, sarcsReportPathNonArray); sarcStats.sarcStats(); log.info("sarc done");