MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

URLConfiguration add FilterType. When filterType is "local" the filtering is applied in argos backend - not throught the API

This commit is contained in:
argirok 2022-03-23 16:03:42 +02:00
parent 116f9b99ea
commit 62b60b93a8
3 changed files with 18 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java
View File

@ -17,6 +17,7 @@ public class UrlConfiguration {
private String firstpage;
private String requestType = "GET";
private String requestBody = "";
private String filterType = "local";
public String getKey() {
return key;
@ -112,6 +113,11 @@ public class UrlConfiguration {
public void setRequestBody(String requestBody) {
this.requestBody = requestBody != null ? requestBody : "";
}
public String getFilterType() {
return filterType;
}
@XmlElement(name = "filterType")
public void setFilterType(String filterType) {
this.filterType = filterType;
}
}

9
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -197,7 +197,7 @@ public class RemoteFetcher {
ifFunderQueryExist(urlConfiguration, externalUrlCriteria);
if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
try {
results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalUrlCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType()));
results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalUrlCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType()));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
@ -277,13 +277,18 @@ public class RemoteFetcher {
return completedPath;
}
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType) throws Exception {
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType) throws Exception {
Set<Integer> pages = new HashSet<>();
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage);
String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage);
Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType);
if(filterType != null && filterType.equals("local") && (externalUrlCriteria.getLike() != null && !externalUrlCriteria.getLike().isEmpty())){
results.setResults(results.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(externalUrlCriteria.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
if (fetchStrategy == FetchStrategy.FIRST)
return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());

5
dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml
View File

@ -299,8 +299,9 @@
<count>'count'</count>
</fields>
</data>
<filterType>local</filterType>
</urlConfig>
<urlConfig>
<!-- <urlConfig>
<key>openaire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
@ -350,7 +351,7 @@
<count>'count'</count>
</fields>
</data>
</urlConfig>
</urlConfig>-->
<!-- <urlConfig>
<key>cristin</key>
<label>Cristin</label>