diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index 53b408323..e19c14aee 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -304,23 +304,12 @@ public class RemoteFetcher { if (response.getHeaders().get("Content-Type").get(0).contains("json")) { DocumentContext jsonContext = JsonPath.parse(response.getBody()); - if (jsonDataPath.getFieldsUrlConfiguration().getSource() != null) { - results = RemoteFetcherUtils.getFromJsonWithSource(jsonContext, jsonDataPath); - } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu - results = RemoteFetcherUtils.getFromJsonWithParsedData(jsonContext, jsonDataPath); - } else if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) { + if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) { results = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType); - } else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) { - results = RemoteFetcherUtils.getFromJsonWithType(jsonContext, jsonDataPath); } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) { results = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath); } else { - results = new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + (jsonDataPath.getFieldsUrlConfiguration().getName() != null ? jsonDataPath.getFieldsUrlConfiguration().getName(): "") - + (jsonDataPath.getFieldsUrlConfiguration().getDescription() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getDescription(): "") - + (jsonDataPath.getFieldsUrlConfiguration().getUri() !=null ? "," + jsonDataPath.getFieldsUrlConfiguration().getUri() : "") - + (jsonDataPath.getFieldsUrlConfiguration().getId() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getId(): "") + "]"), - new HashMap<>(1, 1)); + results = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath); } results.setResults(results.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue))) .collect(Collectors.toList())); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java index a518ca2f7..ca2781f81 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcherUtils.java @@ -1,67 +1,30 @@ package eu.eudat.logic.proxy.fetching; +import com.jayway.jsonpath.Configuration; import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.spi.json.JacksonJsonProvider; import eu.eudat.logic.proxy.config.DataUrlConfiguration; import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.fetching.entities.Results; import net.minidev.json.JSONArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.*; import java.util.stream.Collectors; public class RemoteFetcherUtils { + private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class); - public static Results getFromJsonWithSource(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { - return new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription() - + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() - + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"), + public static Results getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { + return new Results(parseData(jsonContext, jsonDataPath), new HashMap<>(1, 1)); } - public static Results getFromJsonWithParsedData(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { - Results results = new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() - + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"), - new HashMap<>(1, 1)); - List> fixedResults = results.getResults().stream().map(item -> { - for (int i = 0; i < 2; i++) { - String id; - if (i == 0) { - id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", ""); - } else { - id = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", ""); - } - if (!(item.get(id) instanceof String)) { - Object obj = item.get(id); - if (obj instanceof JSONArray) { - JSONArray jarr = (JSONArray) obj; - if (jarr.get(0) instanceof String) { - item.put(id, jarr.get(0).toString()); - } else { - for (int j = 0; j < jarr.size(); j++) { - mapToMap(id, (Map)jarr.get(j), item, i == 1); - } - } - } else { - if (obj instanceof Map) { - mapToMap(id, (Map) obj, item, i == 1); - } else if (obj != null){ - item.put(id, obj.toString()); - } - } - } - } - return item; - }).collect(Collectors.toList()); - - return new Results(fixedResults, new HashMap<>(1, 1)); - } - public static Results getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcher remoteFetcher, String requestBody, String requestType) { - Results results = new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getPath() - + "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"), + Results results = new Results(parseData(jsonContext, jsonDataPath), new HashMap<>(1, 1)); List> multiResults = results.getResults().stream().map(result -> { @@ -74,34 +37,8 @@ public class RemoteFetcherUtils { return new Results(multiResults, new HashMap<>(1, 1)); } - public static Results getFromJsonWithType(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { - List> tempRes = jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName() - + "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]"); - List> finalRes = new ArrayList<>(); - tempRes.forEach(map -> { - Map resMap = new HashMap<>(); - map.forEach((key, value) -> { - if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) { - resMap.put("tags", ((JSONArray) value).toJSONString()); - } else if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getUri().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) { - resMap.put(key, ((JSONArray) value).toJSONString()); - } else { - resMap.put(key, (String) value); - } - }); - - finalRes.add(resMap); - }); - - return new Results(finalRes, - new HashMap<>(1, 1)); - } - public static Results getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { - Results results = new Results(jsonContext.read(jsonDataPath.getPath() - + "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getFirstName() - + "," + jsonDataPath.getFieldsUrlConfiguration().getLastName() + "]"), + Results results = new Results(parseData(jsonContext, jsonDataPath), new HashMap<>(1, 1)); results.getResults().stream().forEach(entry -> { String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", "")); @@ -112,17 +49,45 @@ public class RemoteFetcherUtils { return results; } - private static void mapToMap(String key, Map source, Map destination, boolean isTitle) { - if (source != null) { - String content = source.get("content"); - /*if (isTitle) { - String classId = source.get("classid"); - if (classId.equals("main title")) { - destination.put(key, content); + private static List> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { + List > rawData = jsonContext.read(jsonDataPath.getPath()); + List> parsedData = new ArrayList<>(); + rawData.forEach(stringObjectMap -> { + parsedData.add(new LinkedHashMap<>()); + Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> { + String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1); + Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0); + try { + String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration())); + if (value != null) { + value = value.replace("'", ""); + if (stringObjectMap.containsKey(value)) { + parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri")))); + } + } + } catch (IllegalAccessException | InvocationTargetException e) { + logger.error(e.getLocalizedMessage(), e); } - } else {*/ - destination.put(key, content); -// } + }); + }); + return parsedData; + } + + private static String normalizeValue(Object value, boolean jsonString) { + if (value instanceof JSONArray) { + if (jsonString) { + return ((JSONArray)value).toJSONString(); + } + JSONArray jarr = (JSONArray) value; + if (jarr.get(0) instanceof String) { + return jarr.get(0).toString(); + } else { + return ((Map)jarr.get(0)).get("content"); + + } + } else if (value instanceof Map) { + return ((Map)value).get("content"); } + return value != null ? value.toString() : null; } }