From 015f6e88dfa01c61c252b60384ef790184990445 Mon Sep 17 00:00:00 2001 From: Spyros Zoupanos Date: Tue, 15 Sep 2020 21:41:28 +0300 Subject: [PATCH 1/2] Removing prefix c: from json --- .../oa/graph/usagestats/export/SarcStats.java | 47 +++++++++++++++---- 1 file changed, 38 insertions(+), 9 deletions(-) diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java index 994bea0634..98d8697753 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java @@ -11,6 +11,7 @@ import java.sql.ResultSet; import java.sql.Statement; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.HashSet; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -187,15 +188,9 @@ public class SarcStats { for (Object aJsonArray : jsonArray) { JSONObject jsonObjectRow = (JSONObject) aJsonArray; - - Set jkeys = jsonObjectRow.keySet(); - for (String jkey : jkeys) { - System.out.println("++++> " + jkey); - String[] splitArray = jkey.split("c:"); - jkey = splitArray[splitArray.length - 1]; - System.out.println("New jkey: " + jkey); - } - + renameKeysRecursively("c:", jsonObjectRow); + System.out.println("oooo====> " + jsonObjectRow.toJSONString()); + fin.write(jsonObjectRow.toJSONString().getBytes()); fin.writeChar('\n'); } @@ -259,6 +254,40 @@ public class SarcStats { ConnectDB.getConnection().close(); } + + private void renameKeysRecursively(String prefixToRemove, JSONArray givenJsonObj) throws Exception { + for(Object jjval : givenJsonObj) { + if(jjval instanceof JSONArray) + renameKeysRecursively(prefixToRemove, (JSONArray)jjval); + else if(jjval instanceof JSONObject) + renameKeysRecursively(prefixToRemove, (JSONObject)jjval); + // All other types of vals + else; + } + } + + private void renameKeysRecursively(String prefixToRemove, JSONObject givenJsonObj) throws Exception { + Set jkeys = new HashSet(givenJsonObj.keySet()); + for (String jkey : jkeys) { + System.out.println("++++> " + jkey); + + String[] splitArray = jkey.split(prefixToRemove); + String newJkey = splitArray[splitArray.length - 1]; + System.out.println("New jkey: " + jkey); + + Object jval = givenJsonObj.get(jkey); + System.out.println("jval ===> " + jval.getClass().getName()); + givenJsonObj.remove(jkey); + givenJsonObj.put(newJkey, jval); + + if(jval instanceof JSONObject) + renameKeysRecursively(prefixToRemove, (JSONObject)jval); + + if(jval instanceof JSONArray) { + renameKeysRecursively(prefixToRemove, (JSONArray)jval); + } + } + } private String getJson(String url) throws Exception { // String cred=username+":"+password; From 49de94c4b11e5a1bbeb32f531f78aa574d905efe Mon Sep 17 00:00:00 2001 From: Spyros Zoupanos Date: Tue, 15 Sep 2020 22:32:14 +0300 Subject: [PATCH 2/2] Removing prefix c: from json --- .../oa/graph/usagestats/export/SarcStats.java | 39 +++++++++++++------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java index 98d8697753..30f30ee229 100644 --- a/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java +++ b/dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java @@ -22,6 +22,7 @@ import org.apache.log4j.Logger; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; /** * Created by dpie @@ -66,7 +67,7 @@ public class SarcStats { public void processSarc(String sarcsReportPath) throws Exception { // There was a problem to download the following file -// processARReport(sarcsReportPath, "https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X"); + processARReport(sarcsReportPath, "https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X"); processARReport(sarcsReportPath, "https://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/", "0873-819X"); processARReport(sarcsReportPath, "https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/", "1646-2335"); processARReport(sarcsReportPath, "https://revistas.rcaap.pt/cct/sushiLite/v1_7/", "2182-3030"); @@ -156,8 +157,20 @@ public class SarcStats { * PrintWriter wr = new PrintWriter(new FileWriter("logs/" + simpleDateFormat.format(start.getTime()) + * ".json")); wr.print(text); wr.close(); */ + + System.out.println("AAAAAAAAAAA text " + text); + JSONParser parser = new JSONParser(); - JSONObject jsonObject = (JSONObject) parser.parse(text); + JSONObject jsonObject = null; + try { + jsonObject = (JSONObject) parser.parse(text); + } + // if there is a parsing error continue with the next url + catch(ParseException pe) { + continue; + } + + jsonObject = (JSONObject) jsonObject.get("sc:ReportResponse"); jsonObject = (JSONObject) jsonObject.get("sc:Report"); if (jsonObject == null) { @@ -249,29 +262,29 @@ public class SarcStats { // break; } - preparedStatement.executeBatch(); - ConnectDB.getConnection().commit(); +// preparedStatement.executeBatch(); +// ConnectDB.getConnection().commit(); ConnectDB.getConnection().close(); } - private void renameKeysRecursively(String prefixToRemove, JSONArray givenJsonObj) throws Exception { + private void renameKeysRecursively(String delimiter, JSONArray givenJsonObj) throws Exception { for(Object jjval : givenJsonObj) { if(jjval instanceof JSONArray) - renameKeysRecursively(prefixToRemove, (JSONArray)jjval); + renameKeysRecursively(delimiter, (JSONArray)jjval); else if(jjval instanceof JSONObject) - renameKeysRecursively(prefixToRemove, (JSONObject)jjval); + renameKeysRecursively(delimiter, (JSONObject)jjval); // All other types of vals else; } } - private void renameKeysRecursively(String prefixToRemove, JSONObject givenJsonObj) throws Exception { + private void renameKeysRecursively(String delimiter, JSONObject givenJsonObj) throws Exception { Set jkeys = new HashSet(givenJsonObj.keySet()); for (String jkey : jkeys) { System.out.println("++++> " + jkey); - String[] splitArray = jkey.split(prefixToRemove); + String[] splitArray = jkey.split(delimiter); String newJkey = splitArray[splitArray.length - 1]; System.out.println("New jkey: " + jkey); @@ -281,10 +294,10 @@ public class SarcStats { givenJsonObj.put(newJkey, jval); if(jval instanceof JSONObject) - renameKeysRecursively(prefixToRemove, (JSONObject)jval); + renameKeysRecursively(delimiter, (JSONObject)jval); if(jval instanceof JSONArray) { - renameKeysRecursively(prefixToRemove, (JSONArray)jval); + renameKeysRecursively(delimiter, (JSONArray)jval); } } } @@ -308,10 +321,12 @@ public class SarcStats { return response.toString(); } catch (Exception e) { + // Logging error and silently continuing log.error("Failed to get URL: " + e); System.out.println("Failed to get URL: " + e); // return null; - throw new Exception("Failed to get URL: " + e.toString(), e); +// throw new Exception("Failed to get URL: " + e.toString(), e); } + return ""; } }