diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java index ea079c8d6..640746082 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java @@ -6,6 +6,7 @@ import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import eu.eudat.logic.proxy.fetching.RemoteFetcher; import eu.eudat.logic.services.ApiContext; +import eu.eudat.logic.utilities.helpers.ListHelper; import eu.eudat.models.data.external.ExternalSourcesItemModel; import eu.eudat.models.data.external.FundersExternalSourcesModel; import eu.eudat.models.data.funder.Funder; @@ -16,6 +17,7 @@ import org.springframework.stereotype.Component; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Component public class FunderManager { @@ -47,6 +49,7 @@ public class FunderManager { funders.add(funder); } funders.sort(Comparator.comparing(Funder::getLabel)); + funders = funders.stream().filter(ListHelper.distinctByKey(Funder::getLabel)).collect(Collectors.toList()); return funders; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataFieldsUrlConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataFieldsUrlConfiguration.java index 829e75b88..0323e8011 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataFieldsUrlConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataFieldsUrlConfiguration.java @@ -11,6 +11,7 @@ public class DataFieldsUrlConfiguration { private String uri; private String description; private String source; + private String count; public String getId() { return id; @@ -57,4 +58,13 @@ public class DataFieldsUrlConfiguration { public void setSource(String source) { this.source = source; } + + public String getCount() { + return count; + } + + @XmlElement(name = "count") + public void setCount(String count) { + this.count = count; + } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataPageConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataPageConfiguration.java new file mode 100644 index 000000000..e0592e3f3 --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataPageConfiguration.java @@ -0,0 +1,24 @@ +package eu.eudat.logic.proxy.config; + +import javax.xml.bind.annotation.XmlElement; + +public class DataPageConfiguration { + private String type; + private String pageParam; + + public String getType() { + return type; + } + @XmlElement(name = "type") + public void setType(String type) { + this.type = type; + } + + public String getPageParam() { + return pageParam; + } + @XmlElement(name = "pageparam") + public void setPageParam(String pageParam) { + this.pageParam = pageParam; + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataSearchConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataSearchConfiguration.java new file mode 100644 index 000000000..8ec232197 --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataSearchConfiguration.java @@ -0,0 +1,24 @@ +package eu.eudat.logic.proxy.config; + +import javax.xml.bind.annotation.XmlElement; + +public class DataSearchConfiguration { + private String type; + private String queryParam; + + public String getType() { + return type; + } + @XmlElement(name = "type") + public void setType(String type) { + this.type = type; + } + + public String getQueryParam() { + return queryParam; + } + @XmlElement(name = "queryparam") + public void setQueryParam(String queryParam) { + this.queryParam = queryParam; + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java index 9a9912c5a..3a14277e1 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java @@ -8,10 +8,13 @@ public class UrlConfiguration { private String key; private String label; private Integer ordinal; + private DataSearchConfiguration search; + private DataPageConfiguration page; private String url; private DataUrlConfiguration data; private String type; private String paginationPath; + private String contentType; public String getKey() { return key; @@ -49,6 +52,24 @@ public class UrlConfiguration { this.ordinal = ordinal; } + public DataSearchConfiguration getSearch() { + return search; + } + + @XmlElement(name = "search") + public void setSearch(DataSearchConfiguration search) { + this.search = search; + } + + public DataPageConfiguration getPage() { + return page; + } + + @XmlElement(name = "page") + public void setPage(DataPageConfiguration page) { + this.page = page; + } + public DataUrlConfiguration getData() { return data; } @@ -75,4 +96,13 @@ public class UrlConfiguration { public void setType(String type) { this.type = type; } + + public String getContentType() { + return contentType; + } + + @XmlElement(name = "contenttype") + public void setContentType(String contentType) { + this.contentType = contentType; + } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index b627f8e74..2af700653 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -4,9 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; -import eu.eudat.logic.proxy.config.DataUrlConfiguration; -import eu.eudat.logic.proxy.config.FetchStrategy; -import eu.eudat.logic.proxy.config.UrlConfiguration; +import eu.eudat.logic.proxy.config.*; import eu.eudat.logic.proxy.config.configloaders.ConfigLoader; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; @@ -62,7 +60,7 @@ public class RemoteFetcher { @Cacheable("funders") public List> getFunders(String query) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getFunders().getUrls(); - FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); + FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @@ -130,9 +128,8 @@ public class RemoteFetcher { List> results = new LinkedList<>(); for (UrlConfiguration urlConfig : urlConfigs) { if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { - results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel())); - } - else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { + results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel(), urlConfig.getSearch(), urlConfig.getPage(), urlConfig.getContentType())); + } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query)); } } @@ -140,45 +137,52 @@ public class RemoteFetcher { } - private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key) throws HugeResultSet { - Set pages = new HashSet(); + private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key, DataSearchConfiguration search, DataPageConfiguration pageConfig, String contentType) throws HugeResultSet { + Set pages = new HashSet<>(); - final String searchQuery = (query != null) && !query.isEmpty() ? "&search=" + query : ""; + String searchQuery = ""; + Results results = null; + if (search != null && search.getType() != null && !search.getType().trim().isEmpty() && + search.getQueryParam() != null && !search.getQueryParam().trim().isEmpty() && + query != null && !query.trim().isEmpty()) { + if (search.getType().equals("queryParam")) + searchQuery = "&" + search.getQueryParam() + "=" + query; + } - Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath); + if (pageConfig != null) { + if (pageConfig.getType().equals("queryParam")) + results = getResultsFromUrl(path + "?" + pageConfig.getPageParam() + "=1" + searchQuery, jsonDataPath, jsonPaginationPath, contentType); + } else { + results = getResultsFromUrl(path, jsonDataPath, jsonPaginationPath, contentType); + } if (fetchStrategy == FetchStrategy.FIRST) - return results == null ? new LinkedList<>() : results.getResults().stream().map(x -> { - x.put("tag", key); - return x; - }).collect(Collectors.toList()); + return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList()); - if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set + if (results != null && results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set for (int i = 2; i <= results.getPagination().get("pages"); i++) pages.add(i); Long maxResults = configLoader.getExternalUrls().getMaxresults(); - if ((maxResults > 0) && (results.getPagination().get("count") > maxResults)) + if ((maxResults > 0 && results != null) && (results.getPagination().get("count") > maxResults)) throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); + String finalSearchQuery = searchQuery; Optional optionalResults = pages.parallelStream() - .map(page -> getResultsFromUrl(path + "?page=" + page + searchQuery, jsonDataPath, jsonPaginationPath)) + .map(page -> getResultsFromUrl(path + "?page=" + page + finalSearchQuery, jsonDataPath, jsonPaginationPath, contentType)) .reduce((result1, result2) -> { result1.getResults().addAll(result2.getResults()); return result1; }); - Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results(); + Results remainingResults = optionalResults.orElseGet(Results::new); remainingResults.getResults().addAll(results.getResults()); - return remainingResults.getResults().stream().map(x -> { - x.put("tag", key); - return x; - }).collect(Collectors.toList()); + return remainingResults.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList()); } - private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath) { + private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType) { try { @@ -186,7 +190,7 @@ public class RemoteFetcher { HttpURLConnection con = (HttpURLConnection) url.openConnection(); con.setRequestMethod("GET"); - con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8"); + con.setRequestProperty("Accept", contentType); int responseCode = con.getResponseCode(); if (responseCode == HttpURLConnection.HTTP_OK) { // success @@ -199,8 +203,12 @@ public class RemoteFetcher { + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"), jsonContext.read(jsonPaginationPath)); - } - else { + } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openair.eu + results = new Results(jsonContext.read(jsonDataPath.getPath() + + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"), + new HashMap<>(1, 1)); + } else { results = new Results(jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"), @@ -254,11 +262,12 @@ public class RemoteFetcher { } private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) { - if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid"; - if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description"; - if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri"; - if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name"; - if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count"; return null; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/helpers/ListHelper.java b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/helpers/ListHelper.java new file mode 100644 index 000000000..110fff44d --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/utilities/helpers/ListHelper.java @@ -0,0 +1,14 @@ +package eu.eudat.logic.utilities.helpers; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.function.Predicate; + +public class ListHelper { + + public static Predicate distinctByKey(Function keyExtractor) { + Map seen = new ConcurrentHashMap<>(); + return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null; + } +} diff --git a/dmp-backend/web/src/main/resources/ExternalUrls.xml b/dmp-backend/web/src/main/resources/ExternalUrls.xml index 4235a1b5c..ecadfe4a3 100644 --- a/dmp-backend/web/src/main/resources/ExternalUrls.xml +++ b/dmp-backend/web/src/main/resources/ExternalUrls.xml @@ -123,7 +123,6 @@ - @@ -179,11 +178,9 @@ FIRST - - @@ -240,18 +237,25 @@ FIRST - - + + + + + + + + + @@ -281,27 +285,90 @@ - internal - + servicesOpenAire + 1 - Internal - FunderInternalMockUpData.json + External + https://services.openaire.eu/search/v2/api/publications?&refine=true&fields=relfunder&page=0&size=0&format=json + application/json; charset=utf-8 - $['data'][*]['attributes'] + $['refineResults']['relfunder'][*] - 'pid' 'name' - 'uri' - 'description' + 'id' + 'count' - $['meta']['pagination']['page','pages','count'] + + servicesOpenAire + + 1 + External + https://services.openaire.eu/search/v2/api/datasets?&refine=true&fields=relfunder&page=0&size=0&format=json + application/json; charset=utf-8 + + $['refineResults']['relfunder'][*] + + 'name' + 'id' + 'count' + + + + + servicesOpenAire + + 1 + External + https://services.openaire.eu/search/v2/api/software?&refine=true&fields=relfunder&page=0&size=0&format=json + application/json; charset=utf-8 + + $['refineResults']['relfunder'][*] + + 'name' + 'id' + 'count' + + + + + servicesOpenAire + + 1 + External + https://services.openaire.eu/search/v2/api/other?&refine=true&fields=relfunder&page=0&size=0&format=json + application/json; charset=utf-8 + + $['refineResults']['relfunder'][*] + + 'name' + 'id' + 'count' + + + + + + + + + + + + + + + + + + + + FIRST -