Allow remote fetcher to retrieve data with POST requests and added a new

OpenAire repository source

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java

@@ -15,6 +15,8 @@ public class UrlConfiguration {
     private String contentType;
     private String funderQuery;
     private String firstpage;
+    private String requestType = "GET";
+    private String requestBody = "";
 
     public String getKey() {
         return key;
@@ -95,4 +97,21 @@ public class UrlConfiguration {
     public void setFirstpage(String firstpage) {
         this.firstpage = firstpage;
     }
+    
+    public String getRequestType() {
+        return requestType;
+    }
+    @XmlElement(name = "request")
+    public void setRequestType(String requestType) {
+        this.requestType = requestType != null ? requestType : "GET";
+    }
+	public String getRequestBody() {
+		return requestBody;
+	}
+	@XmlElement(name = "requestBody")
+	public void setRequestBody(String requestBody) {
+		this.requestBody = requestBody != null ? requestBody : "";
+	}
+    
+    
 }

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java

@@ -1,6 +1,7 @@
 package eu.eudat.logic.proxy.fetching;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.jayway.jsonpath.DocumentContext;
 import com.jayway.jsonpath.JsonPath;
@@ -163,7 +164,7 @@ public class RemoteFetcher {
         for (UrlConfiguration urlConfig : urlConfigs) {
             ifFunderQueryExist(urlConfig, externalUrlCriteria);
             if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
-                results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage()));
+                results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType()));
             } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
                 results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
             }
@@ -232,12 +233,13 @@ public class RemoteFetcher {
         return completedPath;
     }
 
-    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage) throws HugeResultSet {
+    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws HugeResultSet {
         Set<Integer> pages = new HashSet<>();
 
         String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage);
+        String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage);
 
-        Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType);
+        Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType);
         if (fetchStrategy == FetchStrategy.FIRST)
             return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
 
@@ -250,7 +252,7 @@ public class RemoteFetcher {
             throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
 
         Optional<Results> optionalResults = pages.parallelStream()
-                .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType))
+                .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType))
                 .reduce((result1, result2) -> {
                     result1.getResults().addAll(result2.getResults());
                     return result1;
@@ -262,12 +264,12 @@ public class RemoteFetcher {
     }
 
 
-    private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType) {
+    private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) {
 
         try {
         	RestTemplate restTemplate = new RestTemplate();
         	HttpHeaders headers = new HttpHeaders();
-        	HttpEntity<String> entity;
+        	HttpEntity<JsonNode> entity;
         	ResponseEntity<String> response;
 			/*
 			 * URL url = new URL(urlString.replaceAll(" ", "%20"));
@@ -277,11 +279,12 @@ public class RemoteFetcher {
 			 */
             if (contentType != null && !contentType.isEmpty()) {
                headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
+               headers.setContentType(MediaType.valueOf(contentType));
             }
-            
+            JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
-            entity = new HttpEntity<>("parameters", headers);
+            entity = new HttpEntity<>(jsonBody, headers);
 
-            response = restTemplate.exchange(urlString, HttpMethod.GET, entity, String.class);
+            response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class);
             if (response.getStatusCode() == HttpStatus.OK) { // success
                 //do here all the parsing
                 Results results = new Results();
@@ -342,7 +345,7 @@ public class RemoteFetcher {
                             externalUrlCriteria.setPath(result.get("path"));
                             externalUrlCriteria.setHost(result.get("host"));
                             String replacedPath = replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage());
-                            return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType());
+                            return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType);
                         }).filter(Objects::nonNull).map(results1 -> results1.results.get(0)).collect(Collectors.toList());
                         results = new Results(multiResults, new HashMap<>(1, 1));
                     } else if (jsonDataPath.getFieldsUrlConfiguration().getTypes() != null) {

dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml

@@ -532,6 +532,26 @@
                 </data>
                 <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
             </urlConfig>
+            <urlConfig>
+                <key>openairealt2</key>
+                <label>Another OpenAIRE Alternative</label>
+                <ordinal>1</ordinal>
+                <type>External</type>
+                <request>POST</request>
+                <url>https://services.openaire.eu/openaire/ds/searchregistered/{page}/{pageSize}?requestSortBy=id&amp;order=ASCENDING</url>
+                <firstPage>0</firstPage>
+                <contenttype>application/json</contenttype>
+                <requestBody>{"officialname": "{like}"}</requestBody>
+                <data>
+                    <path>$['datasourceInfo'][*]</path>
+                    <fields>
+                        <id>'id'</id>
+                        <name>'officialname'</name>
+                        <count>'count'</count>
+                    </fields>
+                </data>
+                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
+            </urlConfig>
             <!--<urlConfig>
                 <key>cristin</key>
                 <label>Cristin</label>