MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Refactor Remote Fetcher's json parser

This commit is contained in:
George Kalampokis 2022-01-19 12:12:20 +02:00
parent 61de8c4df0
commit ce386ac047
2 changed files with 51 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -304,23 +304,12 @@ public class RemoteFetcher {
if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
DocumentContext jsonContext = JsonPath.parse(response.getBody());
if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) {
results = RemoteFetcherUtils.getFromJsonWithSource(jsonContext, jsonDataPath);
} else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu
results = RemoteFetcherUtils.getFromJsonWithParsedData(jsonContext, jsonDataPath);
} else if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
results = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType);
} else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) {
results = RemoteFetcherUtils.getFromJsonWithType(jsonContext, jsonDataPath);
} else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
results = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
} else {
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + (jsonDataPath.getFieldsUrlConfiguration().getName() != null ? jsonDataPath.getFieldsUrlConfiguration().getName(): "")
+ (jsonDataPath.getFieldsUrlConfiguration().getDescription() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getDescription(): "")
+ (jsonDataPath.getFieldsUrlConfiguration().getUri() !=null ? "," + jsonDataPath.getFieldsUrlConfiguration().getUri() : "")
+ (jsonDataPath.getFieldsUrlConfiguration().getId() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getId(): "") + "]"),
new HashMap<>(1, 1));
results = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath);
}
results.setResults(results.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
.collect(Collectors.toList()));

133
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java
View File

@ -1,67 +1,30 @@
package eu.eudat.logic.proxy.fetching;
import com.jayway.jsonpath.Configuration;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
import eu.eudat.logic.proxy.config.DataUrlConfiguration;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.fetching.entities.Results;
import net.minidev.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.*;
import java.util.stream.Collectors;
public class RemoteFetcherUtils {
private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class);
public static Results getFromJsonWithSource(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
return new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
public static Results getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
return new Results(parseData(jsonContext, jsonDataPath),
new HashMap<>(1, 1));
}
public static Results getFromJsonWithParsedData(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
new HashMap<>(1, 1));
List<Map<String, String>> fixedResults = results.getResults().stream().map(item -> {
for (int i = 0; i < 2; i++) {
String id;
if (i == 0) {
id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", "");
} else {
id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", "");
}
if (!(item.get(id) instanceof String)) {
Object obj = item.get(id);
if (obj instanceof JSONArray) {
JSONArray jarr = (JSONArray) obj;
if (jarr.get(0) instanceof String) {
item.put(id, jarr.get(0).toString());
} else {
for (int j = 0; j < jarr.size(); j++) {
mapToMap(id, (Map<String, String>)jarr.get(j), item, i == 1);
}
}
} else {
if (obj instanceof Map) {
mapToMap(id, (Map<String, String>) obj, item, i == 1);
} else if (obj != null){
item.put(id, obj.toString());
}
}
}
}
return item;
}).collect(Collectors.toList());
return new Results(fixedResults, new HashMap<>(1, 1));
}
public static Results getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcher remoteFetcher, String requestBody, String requestType) {
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getPath()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"),
Results results = new Results(parseData(jsonContext, jsonDataPath),
new HashMap<>(1, 1));
List<Map<String, String>> multiResults = results.getResults().stream().map(result -> {
@ -74,34 +37,8 @@ public class RemoteFetcherUtils {
return new Results(multiResults, new HashMap<>(1, 1));
}
public static Results getFromJsonWithType(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
List<Map<String, Object>> tempRes = jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]");
List<Map<String, String>> finalRes = new ArrayList<>();
tempRes.forEach(map -> {
Map<String, String> resMap = new HashMap<>();
map.forEach((key, value) -> {
if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
resMap.put("tags", ((JSONArray) value).toJSONString());
} else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
resMap.put(key, ((JSONArray) value).toJSONString());
} else {
resMap.put(key, (String) value);
}
});
finalRes.add(resMap);
});
return new Results(finalRes,
new HashMap<>(1, 1));
}
public static Results getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"),
Results results = new Results(parseData(jsonContext, jsonDataPath),
new HashMap<>(1, 1));
results.getResults().stream().forEach(entry -> {
String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
@ -112,17 +49,45 @@ public class RemoteFetcherUtils {
return results;
}
private static void mapToMap(String key, Map<String, String> source, Map<String, String> destination, boolean isTitle) {
if (source != null) {
String content = source.get("content");
/*if (isTitle) {
String classId = source.get("classid");
if (classId.equals("main title")) {
destination.put(key, content);
private static List<Map<String, String>> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
List <Map<String, Object>> rawData = jsonContext.read(jsonDataPath.getPath());
List<Map<String, String>> parsedData = new ArrayList<>();
rawData.forEach(stringObjectMap -> {
parsedData.add(new LinkedHashMap<>());
Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> {
String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1);
Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0);
try {
String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration()));
if (value != null) {
value = value.replace("'", "");
if (stringObjectMap.containsKey(value)) {
parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri"))));
}
}
} catch (IllegalAccessException | InvocationTargetException e) {
logger.error(e.getLocalizedMessage(), e);
}
} else {*/
destination.put(key, content);
// }
});
});
return parsedData;
}
private static String normalizeValue(Object value, boolean jsonString) {
if (value instanceof JSONArray) {
if (jsonString) {
return ((JSONArray)value).toJSONString();
}
JSONArray jarr = (JSONArray) value;
if (jarr.get(0) instanceof String) {
return jarr.get(0).toString();
} else {
return ((Map<String, String>)jarr.get(0)).get("content");
}
} else if (value instanceof Map) {
return ((Map<String, String>)value).get("content");
}
return value != null ? value.toString() : null;
}
}