MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Allow remote fetcher to retrieve data with POST requests and added a new 

OpenAire repository source
This commit is contained in:
George Kalampokis 2020-11-06 18:45:20 +02:00
parent eae7f22f72
commit cda31be0bd
3 changed files with 52 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java
View File

@ -15,6 +15,8 @@ public class UrlConfiguration {
private String contentType; private String contentType;
private String funderQuery; private String funderQuery;
private String firstpage; private String firstpage;
private String requestType = "GET";
private String requestBody = "";
public String getKey() { public String getKey() {
return key; return key;
@ -95,4 +97,21 @@ public class UrlConfiguration {
public void setFirstpage(String firstpage) { public void setFirstpage(String firstpage) {
this.firstpage = firstpage; this.firstpage = firstpage;
} }
public String getRequestType() {
return requestType;
}
@XmlElement(name = "request")
public void setRequestType(String requestType) {
this.requestType = requestType != null ? requestType : "GET";
}
public String getRequestBody() {
return requestBody;
}
@XmlElement(name = "requestBody")
public void setRequestBody(String requestBody) {
this.requestBody = requestBody != null ? requestBody : "";
}
} }

23
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -1,6 +1,7 @@
package eu.eudat.logic.proxy.fetching; package eu.eudat.logic.proxy.fetching;
import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.JsonPath;
@ -163,7 +164,7 @@ public class RemoteFetcher {
for (UrlConfiguration urlConfig : urlConfigs) { for (UrlConfiguration urlConfig : urlConfigs) {
ifFunderQueryExist(urlConfig, externalUrlCriteria); ifFunderQueryExist(urlConfig, externalUrlCriteria);
if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage())); results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType()));
} else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike())); results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
} }
@ -232,12 +233,13 @@ public class RemoteFetcher {
return completedPath; return completedPath;
} }
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage) throws HugeResultSet { private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws HugeResultSet {
Set<Integer> pages = new HashSet<>(); Set<Integer> pages = new HashSet<>();
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage); String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage);
String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage);
Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType); Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType);
if (fetchStrategy == FetchStrategy.FIRST) if (fetchStrategy == FetchStrategy.FIRST)
return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
@ -250,7 +252,7 @@ public class RemoteFetcher {
throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
Optional<Results> optionalResults = pages.parallelStream() Optional<Results> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType)) .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType))
.reduce((result1, result2) -> { .reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults()); result1.getResults().addAll(result2.getResults());
return result1; return result1;
@ -262,12 +264,12 @@ public class RemoteFetcher {
} }
private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType) { private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) {
try { try {
RestTemplate restTemplate = new RestTemplate(); RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
HttpEntity<String> entity; HttpEntity<JsonNode> entity;
ResponseEntity<String> response; ResponseEntity<String> response;
/* /*
* URL url = new URL(urlString.replaceAll(" ", "%20")); * URL url = new URL(urlString.replaceAll(" ", "%20"));
@ -277,11 +279,12 @@ public class RemoteFetcher {
*/ */
if (contentType != null && !contentType.isEmpty()) { if (contentType != null && !contentType.isEmpty()) {
headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
headers.setContentType(MediaType.valueOf(contentType));
} }
JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
entity = new HttpEntity<>(jsonBody, headers);
entity = new HttpEntity<>("parameters", headers); response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class);
response = restTemplate.exchange(urlString, HttpMethod.GET, entity, String.class);
if (response.getStatusCode() == HttpStatus.OK) { // success if (response.getStatusCode() == HttpStatus.OK) { // success
//do here all the parsing //do here all the parsing
Results results = new Results(); Results results = new Results();
@ -342,7 +345,7 @@ public class RemoteFetcher {
externalUrlCriteria.setPath(result.get("path")); externalUrlCriteria.setPath(result.get("path"));
externalUrlCriteria.setHost(result.get("host")); externalUrlCriteria.setHost(result.get("host"));
String replacedPath = replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage()); String replacedPath = replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage());
return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType()); return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType);
}).filter(Objects::nonNull).map(results1 -> results1.results.get(0)).collect(Collectors.toList()); }).filter(Objects::nonNull).map(results1 -> results1.results.get(0)).collect(Collectors.toList());
results = new Results(multiResults, new HashMap<>(1, 1)); results = new Results(multiResults, new HashMap<>(1, 1));
} else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) { } else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) {

20
dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml
View File

@ -532,6 +532,26 @@
</data> </data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath> <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig> </urlConfig>
<urlConfig>
<key>openairealt2</key>
<label>Another OpenAIRE Alternative</label>
<ordinal>1</ordinal>
<type>External</type>
<request>POST</request>
<url>https://services.openaire.eu/openaire/ds/searchregistered/{page}/{pageSize}?requestSortBy=id&amp;order=ASCENDING</url>
<firstPage>0</firstPage>
<contenttype>application/json</contenttype>
<requestBody>{"officialname": "{like}"}</requestBody>
<data>
<path>$['datasourceInfo'][*]</path>
<fields>
<id>'id'</id>
<name>'officialname'</name>
<count>'count'</count>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
<!--<urlConfig> <!--<urlConfig>
<key>cristin</key> <key>cristin</key>
<label>Cristin</label> <label>Cristin</label>