Refactor Remote Fetcher's json parser
This commit is contained in:
parent
61de8c4df0
commit
ce386ac047
|
@ -304,23 +304,12 @@ public class RemoteFetcher {
|
||||||
if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
|
if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
|
||||||
DocumentContext jsonContext = JsonPath.parse(response.getBody());
|
DocumentContext jsonContext = JsonPath.parse(response.getBody());
|
||||||
|
|
||||||
if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) {
|
if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
|
||||||
results = RemoteFetcherUtils.getFromJsonWithSource(jsonContext, jsonDataPath);
|
|
||||||
} else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu
|
|
||||||
results = RemoteFetcherUtils.getFromJsonWithParsedData(jsonContext, jsonDataPath);
|
|
||||||
} else if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
|
|
||||||
results = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType);
|
results = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType);
|
||||||
} else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) {
|
|
||||||
results = RemoteFetcherUtils.getFromJsonWithType(jsonContext, jsonDataPath);
|
|
||||||
} else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
|
} else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
|
||||||
results = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
|
results = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
|
||||||
} else {
|
} else {
|
||||||
results = new Results(jsonContext.read(jsonDataPath.getPath()
|
results = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath);
|
||||||
+ "[" + (jsonDataPath.getFieldsUrlConfiguration().getName() != null ? jsonDataPath.getFieldsUrlConfiguration().getName(): "")
|
|
||||||
+ (jsonDataPath.getFieldsUrlConfiguration().getDescription() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getDescription(): "")
|
|
||||||
+ (jsonDataPath.getFieldsUrlConfiguration().getUri() !=null ? "," + jsonDataPath.getFieldsUrlConfiguration().getUri() : "")
|
|
||||||
+ (jsonDataPath.getFieldsUrlConfiguration().getId() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getId(): "") + "]"),
|
|
||||||
new HashMap<>(1, 1));
|
|
||||||
}
|
}
|
||||||
results.setResults(results.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
|
results.setResults(results.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
|
||||||
.collect(Collectors.toList()));
|
.collect(Collectors.toList()));
|
||||||
|
|
|
@ -1,67 +1,30 @@
|
||||||
package eu.eudat.logic.proxy.fetching;
|
package eu.eudat.logic.proxy.fetching;
|
||||||
|
|
||||||
|
import com.jayway.jsonpath.Configuration;
|
||||||
import com.jayway.jsonpath.DocumentContext;
|
import com.jayway.jsonpath.DocumentContext;
|
||||||
|
import com.jayway.jsonpath.spi.json.JacksonJsonProvider;
|
||||||
import eu.eudat.logic.proxy.config.DataUrlConfiguration;
|
import eu.eudat.logic.proxy.config.DataUrlConfiguration;
|
||||||
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
|
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
|
||||||
import eu.eudat.logic.proxy.fetching.entities.Results;
|
import eu.eudat.logic.proxy.fetching.entities.Results;
|
||||||
import net.minidev.json.JSONArray;
|
import net.minidev.json.JSONArray;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import java.lang.reflect.InvocationTargetException;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.stream.Collectors;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
public class RemoteFetcherUtils {
|
public class RemoteFetcherUtils {
|
||||||
|
private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class);
|
||||||
|
|
||||||
public static Results getFromJsonWithSource(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
public static Results getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
||||||
return new Results(jsonContext.read(jsonDataPath.getPath()
|
return new Results(parseData(jsonContext, jsonDataPath),
|
||||||
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
|
|
||||||
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
|
|
||||||
+ "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
|
|
||||||
new HashMap<>(1, 1));
|
new HashMap<>(1, 1));
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Results getFromJsonWithParsedData(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
|
||||||
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
|
|
||||||
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
|
|
||||||
+ "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
|
|
||||||
new HashMap<>(1, 1));
|
|
||||||
List<Map<String, String>> fixedResults = results.getResults().stream().map(item -> {
|
|
||||||
for (int i = 0; i < 2; i++) {
|
|
||||||
String id;
|
|
||||||
if (i == 0) {
|
|
||||||
id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", "");
|
|
||||||
} else {
|
|
||||||
id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", "");
|
|
||||||
}
|
|
||||||
if (!(item.get(id) instanceof String)) {
|
|
||||||
Object obj = item.get(id);
|
|
||||||
if (obj instanceof JSONArray) {
|
|
||||||
JSONArray jarr = (JSONArray) obj;
|
|
||||||
if (jarr.get(0) instanceof String) {
|
|
||||||
item.put(id, jarr.get(0).toString());
|
|
||||||
} else {
|
|
||||||
for (int j = 0; j < jarr.size(); j++) {
|
|
||||||
mapToMap(id, (Map<String, String>)jarr.get(j), item, i == 1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if (obj instanceof Map) {
|
|
||||||
mapToMap(id, (Map<String, String>) obj, item, i == 1);
|
|
||||||
} else if (obj != null){
|
|
||||||
item.put(id, obj.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return item;
|
|
||||||
}).collect(Collectors.toList());
|
|
||||||
|
|
||||||
return new Results(fixedResults, new HashMap<>(1, 1));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Results getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcher remoteFetcher, String requestBody, String requestType) {
|
public static Results getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcher remoteFetcher, String requestBody, String requestType) {
|
||||||
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
|
Results results = new Results(parseData(jsonContext, jsonDataPath),
|
||||||
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getPath()
|
|
||||||
+ "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"),
|
|
||||||
new HashMap<>(1, 1));
|
new HashMap<>(1, 1));
|
||||||
|
|
||||||
List<Map<String, String>> multiResults = results.getResults().stream().map(result -> {
|
List<Map<String, String>> multiResults = results.getResults().stream().map(result -> {
|
||||||
|
@ -74,34 +37,8 @@ public class RemoteFetcherUtils {
|
||||||
return new Results(multiResults, new HashMap<>(1, 1));
|
return new Results(multiResults, new HashMap<>(1, 1));
|
||||||
}
|
}
|
||||||
|
|
||||||
public static Results getFromJsonWithType(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
|
||||||
List<Map<String, Object>> tempRes = jsonContext.read(jsonDataPath.getPath()
|
|
||||||
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName()
|
|
||||||
+ "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]");
|
|
||||||
List<Map<String, String>> finalRes = new ArrayList<>();
|
|
||||||
tempRes.forEach(map -> {
|
|
||||||
Map<String, String> resMap = new HashMap<>();
|
|
||||||
map.forEach((key, value) -> {
|
|
||||||
if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
|
|
||||||
resMap.put("tags", ((JSONArray) value).toJSONString());
|
|
||||||
} else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
|
|
||||||
resMap.put(key, ((JSONArray) value).toJSONString());
|
|
||||||
} else {
|
|
||||||
resMap.put(key, (String) value);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
finalRes.add(resMap);
|
|
||||||
});
|
|
||||||
|
|
||||||
return new Results(finalRes,
|
|
||||||
new HashMap<>(1, 1));
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Results getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
public static Results getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
||||||
Results results = new Results(jsonContext.read(jsonDataPath.getPath()
|
Results results = new Results(parseData(jsonContext, jsonDataPath),
|
||||||
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName()
|
|
||||||
+ "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"),
|
|
||||||
new HashMap<>(1, 1));
|
new HashMap<>(1, 1));
|
||||||
results.getResults().stream().forEach(entry -> {
|
results.getResults().stream().forEach(entry -> {
|
||||||
String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
|
String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
|
||||||
|
@ -112,17 +49,45 @@ public class RemoteFetcherUtils {
|
||||||
return results;
|
return results;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void mapToMap(String key, Map<String, String> source, Map<String, String> destination, boolean isTitle) {
|
private static List<Map<String, String>> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
||||||
if (source != null) {
|
List <Map<String, Object>> rawData = jsonContext.read(jsonDataPath.getPath());
|
||||||
String content = source.get("content");
|
List<Map<String, String>> parsedData = new ArrayList<>();
|
||||||
/*if (isTitle) {
|
rawData.forEach(stringObjectMap -> {
|
||||||
String classId = source.get("classid");
|
parsedData.add(new LinkedHashMap<>());
|
||||||
if (classId.equals("main title")) {
|
Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> {
|
||||||
destination.put(key, content);
|
String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1);
|
||||||
|
Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0);
|
||||||
|
try {
|
||||||
|
String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration()));
|
||||||
|
if (value != null) {
|
||||||
|
value = value.replace("'", "");
|
||||||
|
if (stringObjectMap.containsKey(value)) {
|
||||||
|
parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri"))));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (IllegalAccessException | InvocationTargetException e) {
|
||||||
|
logger.error(e.getLocalizedMessage(), e);
|
||||||
}
|
}
|
||||||
} else {*/
|
});
|
||||||
destination.put(key, content);
|
});
|
||||||
// }
|
return parsedData;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String normalizeValue(Object value, boolean jsonString) {
|
||||||
|
if (value instanceof JSONArray) {
|
||||||
|
if (jsonString) {
|
||||||
|
return ((JSONArray)value).toJSONString();
|
||||||
|
}
|
||||||
|
JSONArray jarr = (JSONArray) value;
|
||||||
|
if (jarr.get(0) instanceof String) {
|
||||||
|
return jarr.get(0).toString();
|
||||||
|
} else {
|
||||||
|
return ((Map<String, String>)jarr.get(0)).get("content");
|
||||||
|
|
||||||
|
}
|
||||||
|
} else if (value instanceof Map) {
|
||||||
|
return ((Map<String, String>)value).get("content");
|
||||||
}
|
}
|
||||||
|
return value != null ? value.toString() : null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue