From cda31be0bdc66fe03888f476926f4e34d2f5dfa3 Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Fri, 6 Nov 2020 18:45:20 +0200 Subject: [PATCH] Allow remote fetcher to retrieve data with POST requests and added a new OpenAire repository source --- .../logic/proxy/config/UrlConfiguration.java | 19 +++++++++++++++ .../logic/proxy/fetching/RemoteFetcher.java | 23 +++++++++++-------- .../resources/externalUrls/ExternalUrls.xml | 20 ++++++++++++++++ 3 files changed, 52 insertions(+), 10 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java index 762153ad4..74a35ed45 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java @@ -15,6 +15,8 @@ public class UrlConfiguration { private String contentType; private String funderQuery; private String firstpage; + private String requestType = "GET"; + private String requestBody = ""; public String getKey() { return key; @@ -95,4 +97,21 @@ public class UrlConfiguration { public void setFirstpage(String firstpage) { this.firstpage = firstpage; } + + public String getRequestType() { + return requestType; + } + @XmlElement(name = "request") + public void setRequestType(String requestType) { + this.requestType = requestType != null ? requestType : "GET"; + } + public String getRequestBody() { + return requestBody; + } + @XmlElement(name = "requestBody") + public void setRequestBody(String requestBody) { + this.requestBody = requestBody != null ? requestBody : ""; + } + + } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index fd5caae87..6f6d366cb 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -1,6 +1,7 @@ package eu.eudat.logic.proxy.fetching; import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; @@ -163,7 +164,7 @@ public class RemoteFetcher { for (UrlConfiguration urlConfig : urlConfigs) { ifFunderQueryExist(urlConfig, externalUrlCriteria); if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { - results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage())); + results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType())); } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike())); } @@ -232,12 +233,13 @@ public class RemoteFetcher { return completedPath; } - private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage) throws HugeResultSet { + private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws HugeResultSet { Set pages = new HashSet<>(); String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage); + String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage); - Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType); + Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType); if (fetchStrategy == FetchStrategy.FIRST) return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); @@ -250,7 +252,7 @@ public class RemoteFetcher { throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); Optional optionalResults = pages.parallelStream() - .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType)) + .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType)) .reduce((result1, result2) -> { result1.getResults().addAll(result2.getResults()); return result1; @@ -262,12 +264,12 @@ public class RemoteFetcher { } - private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType) { + private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) { try { RestTemplate restTemplate = new RestTemplate(); HttpHeaders headers = new HttpHeaders(); - HttpEntity entity; + HttpEntity entity; ResponseEntity response; /* * URL url = new URL(urlString.replaceAll(" ", "%20")); @@ -277,11 +279,12 @@ public class RemoteFetcher { */ if (contentType != null && !contentType.isEmpty()) { headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); + headers.setContentType(MediaType.valueOf(contentType)); } - - entity = new HttpEntity<>("parameters", headers); + JsonNode jsonBody = new ObjectMapper().readTree(requestBody); + entity = new HttpEntity<>(jsonBody, headers); - response = restTemplate.exchange(urlString, HttpMethod.GET, entity, String.class); + response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class); if (response.getStatusCode() == HttpStatus.OK) { // success //do here all the parsing Results results = new Results(); @@ -342,7 +345,7 @@ public class RemoteFetcher { externalUrlCriteria.setPath(result.get("path")); externalUrlCriteria.setHost(result.get("host")); String replacedPath = replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage()); - return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType()); + return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType); }).filter(Objects::nonNull).map(results1 -> results1.results.get(0)).collect(Collectors.toList()); results = new Results(multiResults, new HashMap<>(1, 1)); } else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) { diff --git a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml index da5ed7f65..b1293cc29 100644 --- a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml +++ b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml @@ -532,6 +532,26 @@ $['meta']['pagination']['page','pages','count'] + + openairealt2 + + 1 + External + POST + https://services.openaire.eu/openaire/ds/searchregistered/{page}/{pageSize}?requestSortBy=id&order=ASCENDING + 0 + application/json + {"officialname": "{like}"} + + $['datasourceInfo'][*] + + 'id' + 'officialname' + 'count' + + + $['meta']['pagination']['page','pages','count'] +