Adds external sources on Funder. (Issue #179)

dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java

@@ -6,6 +6,7 @@ import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
 import eu.eudat.logic.proxy.fetching.RemoteFetcher;
 import eu.eudat.logic.services.ApiContext;
+import eu.eudat.logic.utilities.helpers.ListHelper;
 import eu.eudat.models.data.external.ExternalSourcesItemModel;
 import eu.eudat.models.data.external.FundersExternalSourcesModel;
 import eu.eudat.models.data.funder.Funder;
@@ -16,6 +17,7 @@ import org.springframework.stereotype.Component;
 import java.util.Comparator;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 @Component
 public class FunderManager {
@@ -47,6 +49,7 @@ public class FunderManager {
 			funders.add(funder);
 		}
 		funders.sort(Comparator.comparing(Funder::getLabel));
+		funders = funders.stream().filter(ListHelper.distinctByKey(Funder::getLabel)).collect(Collectors.toList());
 		return funders;
 	}
 }

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataFieldsUrlConfiguration.java

@@ -11,6 +11,7 @@ public class DataFieldsUrlConfiguration {
     private String uri;
     private String description;
     private String source;
+    private String count;
 
     public String getId() {
         return id;
@@ -57,4 +58,13 @@ public class DataFieldsUrlConfiguration {
     public void setSource(String source) {
         this.source = source;
     }
+
+    public String getCount() {
+        return count;
+    }
+
+    @XmlElement(name = "count")
+    public void setCount(String count) {
+        this.count = count;
+    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataPageConfiguration.java

@@ -0,0 +1,24 @@
+package eu.eudat.logic.proxy.config;
+
+import javax.xml.bind.annotation.XmlElement;
+
+public class DataPageConfiguration {
+	private String type;
+	private String pageParam;
+
+	public String getType() {
+		return type;
+	}
+	@XmlElement(name = "type")
+	public void setType(String type) {
+		this.type = type;
+	}
+
+	public String getPageParam() {
+		return pageParam;
+	}
+	@XmlElement(name = "pageparam")
+	public void setPageParam(String pageParam) {
+		this.pageParam = pageParam;
+	}
+}

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataSearchConfiguration.java

@@ -0,0 +1,24 @@
+package eu.eudat.logic.proxy.config;
+
+import javax.xml.bind.annotation.XmlElement;
+
+public class DataSearchConfiguration {
+	private String type;
+	private String queryParam;
+
+	public String getType() {
+		return type;
+	}
+	@XmlElement(name = "type")
+	public void setType(String type) {
+		this.type = type;
+	}
+
+	public String getQueryParam() {
+		return queryParam;
+	}
+	@XmlElement(name = "queryparam")
+	public void setQueryParam(String queryParam) {
+		this.queryParam = queryParam;
+	}
+}

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java

@@ -8,10 +8,13 @@ public class UrlConfiguration {
     private String key;
     private String label;
     private Integer ordinal;
+    private DataSearchConfiguration search;
+    private DataPageConfiguration page;
     private String url;
     private DataUrlConfiguration data;
     private String type;
     private String paginationPath;
+    private String contentType;
 
     public String getKey() {
         return key;
@@ -49,6 +52,24 @@ public class UrlConfiguration {
         this.ordinal = ordinal;
     }
 
+    public DataSearchConfiguration getSearch() {
+        return search;
+    }
+
+    @XmlElement(name = "search")
+    public void setSearch(DataSearchConfiguration search) {
+        this.search = search;
+    }
+
+    public DataPageConfiguration getPage() {
+        return page;
+    }
+
+    @XmlElement(name = "page")
+    public void setPage(DataPageConfiguration page) {
+        this.page = page;
+    }
+
     public DataUrlConfiguration getData() {
         return data;
     }
@@ -75,4 +96,13 @@ public class UrlConfiguration {
     public void setType(String type) {
         this.type = type;
     }
+
+    public String getContentType() {
+        return contentType;
+    }
+
+    @XmlElement(name = "contenttype")
+    public void setContentType(String contentType) {
+        this.contentType = contentType;
+    }
 }

dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java

@@ -4,9 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.jayway.jsonpath.DocumentContext;
 import com.jayway.jsonpath.JsonPath;
-import eu.eudat.logic.proxy.config.DataUrlConfiguration;
+import eu.eudat.logic.proxy.config.*;
-import eu.eudat.logic.proxy.config.FetchStrategy;
-import eu.eudat.logic.proxy.config.UrlConfiguration;
 import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
 import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
 import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
@@ -62,7 +60,7 @@ public class RemoteFetcher {
     @Cacheable("funders")
     public List<Map<String, String>> getFunders(String query) throws NoURLFound, HugeResultSet {
         List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
-        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
+        FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode();
         return getAll(urlConfigs, fetchStrategy, query);
     }
 
@@ -130,9 +128,8 @@ public class RemoteFetcher {
         List<Map<String, String>> results = new LinkedList<>();
         for (UrlConfiguration urlConfig : urlConfigs) {
             if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
-                results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel()));
+                results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel(), urlConfig.getSearch(), urlConfig.getPage(), urlConfig.getContentType()));
-            }
+            } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
-            else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
                 results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query));
             }
         }
@@ -140,45 +137,52 @@ public class RemoteFetcher {
     }
 
 
-    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key) throws HugeResultSet {
+    private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key, DataSearchConfiguration search, DataPageConfiguration pageConfig, String contentType) throws HugeResultSet {
-        Set<Integer> pages = new HashSet<Integer>();
+        Set<Integer> pages = new HashSet<>();
 
-        final String searchQuery = (query != null) && !query.isEmpty() ? "&search=" + query : "";
+        String searchQuery = "";
+        Results results = null;
+        if (search != null && search.getType() != null && !search.getType().trim().isEmpty() &&
+                search.getQueryParam() != null && !search.getQueryParam().trim().isEmpty() &&
+                query != null && !query.trim().isEmpty()) {
+            if (search.getType().equals("queryParam"))
+                searchQuery = "&" + search.getQueryParam() + "=" + query;
+        }
 
-        Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath);
+        if (pageConfig != null) {
+            if (pageConfig.getType().equals("queryParam"))
+                results = getResultsFromUrl(path + "?" + pageConfig.getPageParam() + "=1" + searchQuery, jsonDataPath, jsonPaginationPath, contentType);
+        } else {
+            results = getResultsFromUrl(path, jsonDataPath, jsonPaginationPath, contentType);
+        }
 
         if (fetchStrategy == FetchStrategy.FIRST)
-            return results == null ? new LinkedList<>() : results.getResults().stream().map(x -> {
+            return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
-                x.put("tag", key);
-                return x;
-            }).collect(Collectors.toList());
 
-        if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
+        if (results != null && results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
             for (int i = 2; i <= results.getPagination().get("pages"); i++)
                 pages.add(i);
 
         Long maxResults = configLoader.getExternalUrls().getMaxresults();
-        if ((maxResults > 0) && (results.getPagination().get("count") > maxResults))
+        if ((maxResults > 0 && results != null) && (results.getPagination().get("count") > maxResults))
             throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
 
+        String finalSearchQuery = searchQuery;
         Optional<Results> optionalResults = pages.parallelStream()
-                .map(page -> getResultsFromUrl(path + "?page=" + page + searchQuery, jsonDataPath, jsonPaginationPath))
+                .map(page -> getResultsFromUrl(path + "?page=" + page + finalSearchQuery, jsonDataPath, jsonPaginationPath, contentType))
                 .reduce((result1, result2) -> {
                     result1.getResults().addAll(result2.getResults());
                     return result1;
                 });
-        Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results();
+        Results remainingResults = optionalResults.orElseGet(Results::new);
 
         remainingResults.getResults().addAll(results.getResults());
 
-        return remainingResults.getResults().stream().map(x -> {
+        return remainingResults.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
-            x.put("tag", key);
-            return x;
-        }).collect(Collectors.toList());
     }
 
 
-    private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath) {
+    private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType) {
 
         try {
 
@@ -186,7 +190,7 @@ public class RemoteFetcher {
 
             HttpURLConnection con = (HttpURLConnection) url.openConnection();
             con.setRequestMethod("GET");
-            con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8");
+            con.setRequestProperty("Accept", contentType);
 
             int responseCode = con.getResponseCode();
             if (responseCode == HttpURLConnection.HTTP_OK) { // success
@@ -199,8 +203,12 @@ public class RemoteFetcher {
                             + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
                             + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
                             jsonContext.read(jsonPaginationPath));
-                }
+                } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openair.eu
-                else {
+                    results = new Results(jsonContext.read(jsonDataPath.getPath()
+                            + "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
+                            + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
+                            new HashMap<>(1, 1));
+                } else {
                     results = new Results(jsonContext.read(jsonDataPath.getPath()
                             + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
                             + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
@@ -254,11 +262,12 @@ public class RemoteFetcher {
     }
 
     private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
-        if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null &&  key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid";
-        if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
-        if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null &&  key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
-        if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null &&  key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
-        if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
+        if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null &&  key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
         return null;
     }
 

dmp-backend/web/src/main/java/eu/eudat/logic/utilities/helpers/ListHelper.java

@@ -0,0 +1,14 @@
+package eu.eudat.logic.utilities.helpers;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.function.Function;
+import java.util.function.Predicate;
+
+public class ListHelper {
+
+	public static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
+		Map<Object, Boolean> seen = new ConcurrentHashMap<>();
+		return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
+	}
+}

dmp-backend/web/src/main/resources/ExternalUrls.xml

@@ -123,7 +123,6 @@
     </tags>
 
     <grants>
-
         <urls>
 <!--            <urlConfig>-->
 <!--                <key>cristin</key>-->
@@ -179,11 +178,9 @@
         </urls>
 
         <fetchMode>FIRST</fetchMode>  <!-- EITHER 'FIRST' OR 'ALL' -->
-
     </grants>
 
     <projects>
-
         <urls>
 <!--            <urlConfig>-->
 <!--                <key>cristin</key>-->
@@ -240,18 +237,25 @@
         </urls>
 
         <fetchMode>FIRST</fetchMode>  <!-- EITHER 'FIRST' OR 'ALL' -->
-
     </projects>
 
     <funders>
-
         <urls>
 <!--            <urlConfig>-->
 <!--                <key>cristin</key>-->
 <!--                <label>Cristin</label>-->
 <!--                <ordinal>1</ordinal>-->
+<!--                <search>-->
+<!--                    <type>queryParam</type>-->
+<!--                    <queryparam>search</queryparam>-->
+<!--                </search>-->
+<!--                <page>-->
+<!--                    <type>queryParam</type>-->
+<!--                    <pageparam>page</pageparam>-->
+<!--                </page>-->
 <!--                <type>External</type>-->
 <!--                <url>https://eestore.paas2.uninett.no/api/projectrepo/</url>-->
+<!--                <contenttype>application/vnd.api+json; charset=utf-8</contenttype>-->
 <!--                <data>-->
 <!--                    <path>$['data'][*]['attributes']</path>-->
 <!--                    <fields>-->
@@ -281,27 +285,90 @@
 <!--                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
 <!--            </urlConfig>-->
             <urlConfig>
-                <key>internal</key>
+                <key>servicesOpenAire</key>
-                <label>Internal</label>
+                <label>OpenAIRE</label>
                 <ordinal>1</ordinal>
-                <type>Internal</type>
+                <type>External</type>
-                <url>FunderInternalMockUpData.json</url>
+                <url>https://services.openaire.eu/search/v2/api/publications?&amp;refine=true&amp;fields=relfunder&amp;page=0&amp;size=0&amp;format=json</url>
+                <contenttype>application/json; charset=utf-8</contenttype>
                 <data>
-                    <path>$['data'][*]['attributes']</path>
+                    <path>$['refineResults']['relfunder'][*]</path>
                     <fields>
-                        <id>'pid'</id>
                         <name>'name'</name>
-                        <uri>'uri'</uri>
+                        <id>'id'</id>
-                        <description>'description'</description>
+                        <count>'count'</count>
                     </fields>
                 </data>
-                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
             </urlConfig>
+            <urlConfig>
+                <key>servicesOpenAire</key>
+                <label>OpenAIRE</label>
+                <ordinal>1</ordinal>
+                <type>External</type>
+                <url>https://services.openaire.eu/search/v2/api/datasets?&amp;refine=true&amp;fields=relfunder&amp;page=0&amp;size=0&amp;format=json</url>
+                <contenttype>application/json; charset=utf-8</contenttype>
+                <data>
+                    <path>$['refineResults']['relfunder'][*]</path>
+                    <fields>
+                        <name>'name'</name>
+                        <id>'id'</id>
+                        <count>'count'</count>
+                    </fields>
+                </data>
+            </urlConfig>
+            <urlConfig>
+                <key>servicesOpenAire</key>
+                <label>OpenAIRE</label>
+                <ordinal>1</ordinal>
+                <type>External</type>
+                <url>https://services.openaire.eu/search/v2/api/software?&amp;refine=true&amp;fields=relfunder&amp;page=0&amp;size=0&amp;format=json</url>
+                <contenttype>application/json; charset=utf-8</contenttype>
+                <data>
+                    <path>$['refineResults']['relfunder'][*]</path>
+                    <fields>
+                        <name>'name'</name>
+                        <id>'id'</id>
+                        <count>'count'</count>
+                    </fields>
+                </data>
+            </urlConfig>
+            <urlConfig>
+                <key>servicesOpenAire</key>
+                <label>OpenAIRE</label>
+                <ordinal>1</ordinal>
+                <type>External</type>
+                <url>https://services.openaire.eu/search/v2/api/other?&amp;refine=true&amp;fields=relfunder&amp;page=0&amp;size=0&amp;format=json</url>
+                <contenttype>application/json; charset=utf-8</contenttype>
+                <data>
+                    <path>$['refineResults']['relfunder'][*]</path>
+                    <fields>
+                        <name>'name'</name>
+                        <id>'id'</id>
+                        <count>'count'</count>
+                    </fields>
+                </data>
+            </urlConfig>
+<!--            <urlConfig>-->
+<!--                <key>internal</key>-->
+<!--                <label>Internal</label>-->
+<!--                <ordinal>1</ordinal>-->
+<!--                <type>Internal</type>-->
+<!--                <url>FunderInternalMockUpData.json</url>-->
+<!--                <data>-->
+<!--                    <path>$['data'][*]['attributes']</path>-->
+<!--                    <fields>-->
+<!--                        <id>'pid'</id>-->
+<!--                        <name>'name'</name>-->
+<!--                        <uri>'uri'</uri>-->
+<!--                        <description>'description'</description>-->
+<!--                    </fields>-->
+<!--                </data>-->
+<!--                <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
+<!--            </urlConfig>-->
 
         </urls>
 
         <fetchMode>FIRST</fetchMode>  <!-- EITHER 'FIRST' OR 'ALL' -->
-
     </funders>
 
     <repositories>