spyros
/
dnet-hadoop
forked from D-Net/dnet-hadoop
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Removing prefix c: from json

This commit is contained in:
Spyros Zoupanos 2020-09-15 22:32:14 +03:00
parent 015f6e88df
commit 49de94c4b1
1 changed files with 27 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
dhp-workflows/dhp-usage-stats-update/src/main/java/eu/dnetlib/oa/graph/usagestats/export/SarcStats.java
View File

@ -22,6 +22,7 @@ import org.apache.log4j.Logger;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
/**
* Created by dpie
@ -66,7 +67,7 @@ public class SarcStats {
public void processSarc(String sarcsReportPath) throws Exception {
// There was a problem to download the following file
// processARReport(sarcsReportPath, "https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X");
processARReport(sarcsReportPath, "https://revistas.rcaap.pt/motricidade/sushiLite/v1_7/", "1646-107X");
processARReport(sarcsReportPath, "https://revistas.rcaap.pt/antropologicas/sushiLite/v1_7/", "0873-819X");
processARReport(sarcsReportPath, "https://revistas.rcaap.pt/interaccoes/sushiLite/v1_7/", "1646-2335");
processARReport(sarcsReportPath, "https://revistas.rcaap.pt/cct/sushiLite/v1_7/", "2182-3030");
@ -156,8 +157,20 @@ public class SarcStats {
* PrintWriter wr = new PrintWriter(new FileWriter("logs/" + simpleDateFormat.format(start.getTime()) +
* ".json")); wr.print(text); wr.close();
*/
System.out.println("AAAAAAAAAAA text " + text);
JSONParser parser = new JSONParser();
JSONObject jsonObject = (JSONObject) parser.parse(text);
JSONObject jsonObject = null;
try {
jsonObject = (JSONObject) parser.parse(text);
}
// if there is a parsing error continue with the next url
catch(ParseException pe) {
continue;
}
jsonObject = (JSONObject) jsonObject.get("sc:ReportResponse");
jsonObject = (JSONObject) jsonObject.get("sc:Report");
if (jsonObject == null) {
@ -249,29 +262,29 @@ public class SarcStats {
// break;
}
preparedStatement.executeBatch();
ConnectDB.getConnection().commit();
// preparedStatement.executeBatch();
// ConnectDB.getConnection().commit();
ConnectDB.getConnection().close();
}
private void renameKeysRecursively(String prefixToRemove, JSONArray givenJsonObj) throws Exception {
private void renameKeysRecursively(String delimiter, JSONArray givenJsonObj) throws Exception {
for(Object jjval : givenJsonObj) {
if(jjval instanceof JSONArray)
renameKeysRecursively(prefixToRemove, (JSONArray)jjval);
renameKeysRecursively(delimiter, (JSONArray)jjval);
else if(jjval instanceof JSONObject)
renameKeysRecursively(prefixToRemove, (JSONObject)jjval);
renameKeysRecursively(delimiter, (JSONObject)jjval);
// All other types of vals
else;
}
}
private void renameKeysRecursively(String prefixToRemove, JSONObject givenJsonObj) throws Exception {
private void renameKeysRecursively(String delimiter, JSONObject givenJsonObj) throws Exception {
Set<String> jkeys = new HashSet<String>(givenJsonObj.keySet());
for (String jkey : jkeys) {
System.out.println("++++> " + jkey);
String[] splitArray = jkey.split(prefixToRemove);
String[] splitArray = jkey.split(delimiter);
String newJkey = splitArray[splitArray.length - 1];
System.out.println("New jkey: " + jkey);
@ -281,10 +294,10 @@ public class SarcStats {
givenJsonObj.put(newJkey, jval);
if(jval instanceof JSONObject)
renameKeysRecursively(prefixToRemove, (JSONObject)jval);
renameKeysRecursively(delimiter, (JSONObject)jval);
if(jval instanceof JSONArray) {
renameKeysRecursively(prefixToRemove, (JSONArray)jval);
renameKeysRecursively(delimiter, (JSONArray)jval);
}
}
}
@ -308,10 +321,12 @@ public class SarcStats {
return response.toString();
} catch (Exception e) {
// Logging error and silently continuing
log.error("Failed to get URL: " + e);
System.out.println("Failed to get URL: " + e);
// return null;
throw new Exception("Failed to get URL: " + e.toString(), e);
// throw new Exception("Failed to get URL: " + e.toString(), e);
}
return "";
}
}