MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Adds external sources on Funder. (Issue #179)

This commit is contained in:
gkolokythas 2019-10-01 13:19:39 +03:00
parent 47c2940517
commit 61a1ea9274
8 changed files with 228 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java
View File

@ -6,6 +6,7 @@ import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.proxy.fetching.RemoteFetcher;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.utilities.helpers.ListHelper;
import eu.eudat.models.data.external.ExternalSourcesItemModel;
import eu.eudat.models.data.external.FundersExternalSourcesModel;
import eu.eudat.models.data.funder.Funder;
@ -16,6 +17,7 @@ import org.springframework.stereotype.Component;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Component
public class FunderManager {
@ -47,6 +49,7 @@ public class FunderManager {
funders.add(funder);
}
funders.sort(Comparator.comparing(Funder::getLabel));
funders = funders.stream().filter(ListHelper.distinctByKey(Funder::getLabel)).collect(Collectors.toList());
return funders;
}
}

10
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataFieldsUrlConfiguration.java
View File

@ -11,6 +11,7 @@ public class DataFieldsUrlConfiguration {
private String uri;
private String description;
private String source;
private String count;
public String getId() {
return id;
@ -57,4 +58,13 @@ public class DataFieldsUrlConfiguration {
public void setSource(String source) {
this.source = source;
}
public String getCount() {
return count;
}
@XmlElement(name = "count")
public void setCount(String count) {
this.count = count;
}
}

24
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataPageConfiguration.java Normal file
View File

@ -0,0 +1,24 @@
package eu.eudat.logic.proxy.config;
import javax.xml.bind.annotation.XmlElement;
public class DataPageConfiguration {
private String type;
private String pageParam;
public String getType() {
return type;
}
@XmlElement(name = "type")
public void setType(String type) {
this.type = type;
}
public String getPageParam() {
return pageParam;
}
@XmlElement(name = "pageparam")
public void setPageParam(String pageParam) {
this.pageParam = pageParam;
}
}

24
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/DataSearchConfiguration.java Normal file
View File

@ -0,0 +1,24 @@
package eu.eudat.logic.proxy.config;
import javax.xml.bind.annotation.XmlElement;
public class DataSearchConfiguration {
private String type;
private String queryParam;
public String getType() {
return type;
}
@XmlElement(name = "type")
public void setType(String type) {
this.type = type;
}
public String getQueryParam() {
return queryParam;
}
@XmlElement(name = "queryparam")
public void setQueryParam(String queryParam) {
this.queryParam = queryParam;
}
}

30
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java
View File

@ -8,10 +8,13 @@ public class UrlConfiguration {
private String key;
private String label;
private Integer ordinal;
private DataSearchConfiguration search;
private DataPageConfiguration page;
private String url;
private DataUrlConfiguration data;
private String type;
private String paginationPath;
private String contentType;
public String getKey() {
return key;
@ -49,6 +52,24 @@ public class UrlConfiguration {
this.ordinal = ordinal;
}
public DataSearchConfiguration getSearch() {
return search;
}
@XmlElement(name = "search")
public void setSearch(DataSearchConfiguration search) {
this.search = search;
}
public DataPageConfiguration getPage() {
return page;
}
@XmlElement(name = "page")
public void setPage(DataPageConfiguration page) {
this.page = page;
}
public DataUrlConfiguration getData() {
return data;
}
@ -75,4 +96,13 @@ public class UrlConfiguration {
public void setType(String type) {
this.type = type;
}
public String getContentType() {
return contentType;
}
@XmlElement(name = "contenttype")
public void setContentType(String contentType) {
this.contentType = contentType;
}
}

73
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -4,9 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.logic.proxy.config.DataUrlConfiguration;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
import eu.eudat.logic.proxy.config.*;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
@ -62,7 +60,7 @@ public class RemoteFetcher {
@Cacheable("funders")
public List<Map<String, String>> getFunders(String query) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@ -130,9 +128,8 @@ public class RemoteFetcher {
List<Map<String, String>> results = new LinkedList<>();
for (UrlConfiguration urlConfig : urlConfigs) {
if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel()));
}
else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel(), urlConfig.getSearch(), urlConfig.getPage(), urlConfig.getContentType()));
} else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query));
}
}
@ -140,45 +137,52 @@ public class RemoteFetcher {
}
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key) throws HugeResultSet {
Set<Integer> pages = new HashSet<Integer>();
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key, DataSearchConfiguration search, DataPageConfiguration pageConfig, String contentType) throws HugeResultSet {
Set<Integer> pages = new HashSet<>();
final String searchQuery = (query != null) && !query.isEmpty() ? "&search=" + query : "";
String searchQuery = "";
Results results = null;
if (search != null && search.getType() != null && !search.getType().trim().isEmpty() &&
search.getQueryParam() != null && !search.getQueryParam().trim().isEmpty() &&
query != null && !query.trim().isEmpty()) {
if (search.getType().equals("queryParam"))
searchQuery = "&" + search.getQueryParam() + "=" + query;
}
Results results = getResultsFromUrl(path + "?page=1" + searchQuery, jsonDataPath, jsonPaginationPath);
if (pageConfig != null) {
if (pageConfig.getType().equals("queryParam"))
results = getResultsFromUrl(path + "?" + pageConfig.getPageParam() + "=1" + searchQuery, jsonDataPath, jsonPaginationPath, contentType);
} else {
results = getResultsFromUrl(path, jsonDataPath, jsonPaginationPath, contentType);
}
if (fetchStrategy == FetchStrategy.FIRST)
return results == null ? new LinkedList<>() : results.getResults().stream().map(x -> {
x.put("tag", key);
return x;
}).collect(Collectors.toList());
return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
if (results != null && results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
for (int i = 2; i <= results.getPagination().get("pages"); i++)
pages.add(i);
Long maxResults = configLoader.getExternalUrls().getMaxresults();
if ((maxResults > 0) && (results.getPagination().get("count") > maxResults))
if ((maxResults > 0 && results != null) && (results.getPagination().get("count") > maxResults))
throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
String finalSearchQuery = searchQuery;
Optional<Results> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(path + "?page=" + page + searchQuery, jsonDataPath, jsonPaginationPath))
.map(page -> getResultsFromUrl(path + "?page=" + page + finalSearchQuery, jsonDataPath, jsonPaginationPath, contentType))
.reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults());
return result1;
});
Results remainingResults = optionalResults.isPresent() ? optionalResults.get() : new Results();
Results remainingResults = optionalResults.orElseGet(Results::new);
remainingResults.getResults().addAll(results.getResults());
return remainingResults.getResults().stream().map(x -> {
x.put("tag", key);
return x;
}).collect(Collectors.toList());
return remainingResults.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
}
private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath) {
private Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType) {
try {
@ -186,7 +190,7 @@ public class RemoteFetcher {
HttpURLConnection con = (HttpURLConnection) url.openConnection();
con.setRequestMethod("GET");
con.setRequestProperty("Accept", "application/vnd.api+json; charset=utf-8");
con.setRequestProperty("Accept", contentType);
int responseCode = con.getResponseCode();
if (responseCode == HttpURLConnection.HTTP_OK) { // success
@ -199,8 +203,12 @@ public class RemoteFetcher {
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
jsonContext.read(jsonPaginationPath));
}
else {
} else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openair.eu
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
new HashMap<>(1, 1));
} else {
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
@ -254,11 +262,12 @@ public class RemoteFetcher {
}
private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid";
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
if (key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid";
if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
return null;
}

14
dmp-backend/web/src/main/java/eu/eudat/logic/utilities/helpers/ListHelper.java Normal file
View File

@ -0,0 +1,14 @@
package eu.eudat.logic.utilities.helpers;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.function.Function;
import java.util.function.Predicate;
public class ListHelper {
public static <T> Predicate<T> distinctByKey(Function<? super T, ?> keyExtractor) {
Map<Object, Boolean> seen = new ConcurrentHashMap<>();
return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
}
}

97
dmp-backend/web/src/main/resources/ExternalUrls.xml
View File

@ -123,7 +123,6 @@
</tags>
<grants>
<urls>
<!-- <urlConfig>-->
<!-- <key>cristin</key>-->
@ -179,11 +178,9 @@
</urls>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</grants>
<projects>
<urls>
<!-- <urlConfig>-->
<!-- <key>cristin</key>-->
@ -240,18 +237,25 @@
</urls>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</projects>
<funders>
<urls>
<!-- <urlConfig>-->
<!-- <key>cristin</key>-->
<!-- <label>Cristin</label>-->
<!-- <ordinal>1</ordinal>-->
<!-- <search>-->
<!-- <type>queryParam</type>-->
<!-- <queryparam>search</queryparam>-->
<!-- </search>-->
<!-- <page>-->
<!-- <type>queryParam</type>-->
<!-- <pageparam>page</pageparam>-->
<!-- </page>-->
<!-- <type>External</type>-->
<!-- <url>https://eestore.paas2.uninett.no/api/projectrepo/</url>-->
<!-- <contenttype>application/vnd.api+json; charset=utf-8</contenttype>-->
<!-- <data>-->
<!-- <path>$['data'][*]['attributes']</path>-->
<!-- <fields>-->
@ -281,27 +285,90 @@
<!-- <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
<!-- </urlConfig>-->
<urlConfig>
<key>internal</key>
<label>Internal</label>
<key>servicesOpenAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>Internal</type>
<url>FunderInternalMockUpData.json</url>
<type>External</type>
<url>https://services.openaire.eu/search/v2/api/publications?&amp;refine=true&amp;fields=relfunder&amp;page=0&amp;size=0&amp;format=json</url>
<contenttype>application/json; charset=utf-8</contenttype>
<data>
<path>$['data'][*]['attributes']</path>
<path>$['refineResults']['relfunder'][*]</path>
<fields>
<id>'pid'</id>
<name>'name'</name>
<uri>'uri'</uri>
<description>'description'</description>
<id>'id'</id>
<count>'count'</count>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
<urlConfig>
<key>servicesOpenAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://services.openaire.eu/search/v2/api/datasets?&amp;refine=true&amp;fields=relfunder&amp;page=0&amp;size=0&amp;format=json</url>
<contenttype>application/json; charset=utf-8</contenttype>
<data>
<path>$['refineResults']['relfunder'][*]</path>
<fields>
<name>'name'</name>
<id>'id'</id>
<count>'count'</count>
</fields>
</data>
</urlConfig>
<urlConfig>
<key>servicesOpenAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://services.openaire.eu/search/v2/api/software?&amp;refine=true&amp;fields=relfunder&amp;page=0&amp;size=0&amp;format=json</url>
<contenttype>application/json; charset=utf-8</contenttype>
<data>
<path>$['refineResults']['relfunder'][*]</path>
<fields>
<name>'name'</name>
<id>'id'</id>
<count>'count'</count>
</fields>
</data>
</urlConfig>
<urlConfig>
<key>servicesOpenAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://services.openaire.eu/search/v2/api/other?&amp;refine=true&amp;fields=relfunder&amp;page=0&amp;size=0&amp;format=json</url>
<contenttype>application/json; charset=utf-8</contenttype>
<data>
<path>$['refineResults']['relfunder'][*]</path>
<fields>
<name>'name'</name>
<id>'id'</id>
<count>'count'</count>
</fields>
</data>
</urlConfig>
<!-- <urlConfig>-->
<!-- <key>internal</key>-->
<!-- <label>Internal</label>-->
<!-- <ordinal>1</ordinal>-->
<!-- <type>Internal</type>-->
<!-- <url>FunderInternalMockUpData.json</url>-->
<!-- <data>-->
<!-- <path>$['data'][*]['attributes']</path>-->
<!-- <fields>-->
<!-- <id>'pid'</id>-->
<!-- <name>'name'</name>-->
<!-- <uri>'uri'</uri>-->
<!-- <description>'description'</description>-->
<!-- </fields>-->
<!-- </data>-->
<!-- <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
<!-- </urlConfig>-->
</urls>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</funders>
<repositories>