MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Refactors external url fetching to support more complex APIs and adds requested external url on "Grant". (Issue #179)

This commit is contained in:
gkolokythas 2019-10-03 14:06:44 +03:00
parent 9841b495c1
commit dc7f05ffd2
16 changed files with 288 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/GrantDaoImpl.java
View File

@ -11,6 +11,7 @@ import eu.eudat.types.grant.GrantStateType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
import schemasMicrosoftComOfficeOffice.LeftDocument;
import javax.persistence.criteria.JoinType;
import java.util.Date;
@ -51,6 +52,8 @@ public class GrantDaoImpl extends DatabaseAccess<Grant> implements GrantDao {
}
if (criteria.getFunderId() != null && !criteria.getFunderId().trim().isEmpty())
query.where((builder, root) -> builder.equal(root.get("funder").get("id"), UUID.fromString(criteria.getFunderId())));
if (criteria.getFunderReference() != null && !criteria.getFunderReference().isEmpty())
query.where((builder, root) -> builder.equal(root.join("funder", JoinType.LEFT).get("reference"), criteria.getFunderReference()));
query.where((builder, root) -> builder.notEqual(root.get("status"), Grant.Status.DELETED.getValue()));
return query;
}

4
dmp-backend/web/src/main/java/eu/eudat/controllers/Organisations.java
View File

@ -2,6 +2,7 @@ package eu.eudat.controllers;
import eu.eudat.data.query.items.table.organisations.OrganisationsTableRequest;
import eu.eudat.logic.managers.OrganisationsManager;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.services.ApiContext;
@ -38,7 +39,8 @@ public class Organisations extends BaseController {
ResponseEntity<ResponseItem<OrganisationsExternalSourcesModel>> listExternalOrganisations(
@RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type
) throws HugeResultSet, NoURLFound {
List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getOrganisations(query, type);
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getOrganisations(externalUrlCriteria, type);
OrganisationsExternalSourcesModel organisationsExternalSourcesModel = new OrganisationsExternalSourcesModel().fromExternalItem(remoteRepos);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<OrganisationsExternalSourcesModel>().payload(organisationsExternalSourcesModel).status(ApiMessageCode.NO_MESSAGE));
}

4
dmp-backend/web/src/main/java/eu/eudat/controllers/TagController.java
View File

@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.elastic.criteria.TagCriteria;
import eu.eudat.elastic.entities.Dataset;
import eu.eudat.elastic.repository.Repository;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.services.ApiContext;
@ -47,7 +48,8 @@ public class TagController extends BaseController {
public @ResponseBody
ResponseEntity<ResponseItem<TagExternalSourcesModel>> listExternalTagModel(
@RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type) throws HugeResultSet, NoURLFound, IOException {
List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(query, type);
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(externalUrlCriteria, type);
TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos);
// ObjectMapper mapper = new ObjectMapper();

1
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
View File

@ -659,7 +659,6 @@ public class DataManagementPlanManager {
eu.eudat.data.entities.Funder funderEntity = funderDao.getWithCritetia(criteria).getSingleOrDefault();
if (funderEntity != null) funder.setId(funderEntity.getId());
else {
// funder.setType(Funder.FunderType.EXTERNAL.getValue());
funderDao.createOrUpdate(funder);
}
}

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataRepositoryManager.java
View File

@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.dao.criteria.DataRepositoryCriteria;
import eu.eudat.data.entities.DataRepository;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.services.ApiContext;
@ -35,7 +36,8 @@ public class DataRepositoryManager {
}
public List<DataRepositoryModel> getDataRepositories(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRepositories(query, type);
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRepositories(externalUrlCriteria, type);
DataRepositoryCriteria criteria = new DataRepositoryCriteria();
if (!query.isEmpty()) criteria.setLike(query);

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java
View File

@ -2,6 +2,7 @@ package eu.eudat.logic.managers;
import eu.eudat.data.query.items.item.funder.FunderCriteriaRequest;
import eu.eudat.logic.builders.model.models.FunderBuilder;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.proxy.fetching.RemoteFetcher;
@ -37,7 +38,8 @@ public class FunderManager {
QueryableList<eu.eudat.data.entities.Funder> items = apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(funderCriteria.getCriteria());
QueryableList<eu.eudat.data.entities.Funder> authItems = apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getAuthenticated(items, userInfo);
List<Funder> funders = authItems.select(item -> new eu.eudat.models.data.funder.Funder().fromDataModel(item));
List<Map<String, String>> remoteRepos = remoteFetcher.getFunders(funderCriteria.getCriteria().getLike());
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(funderCriteria.getCriteria().getLike());
List<Map<String, String>> remoteRepos = remoteFetcher.getFunders(externalUrlCriteria);
FundersExternalSourcesModel fundersExternalSourcesModel = new FundersExternalSourcesModel().fromExternalItem(remoteRepos);
for (ExternalSourcesItemModel externalListingItem : fundersExternalSourcesModel) {
eu.eudat.models.data.funder.Funder funder = apiContext.getOperationsContext().getBuilderFactory().getBuilder(FunderBuilder.class)

11
dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java
View File

@ -10,7 +10,9 @@ import eu.eudat.data.dao.entities.GrantDao;
import eu.eudat.data.entities.Content;
import eu.eudat.data.entities.DMP;
import eu.eudat.exceptions.files.TempFileNotFoundException;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.services.operations.DatabaseRepository;
import eu.eudat.logic.utilities.helpers.ListHelper;
import eu.eudat.models.data.external.ExternalSourcesItemModel;
import eu.eudat.models.data.external.GrantsExternalSourcesModel;
import eu.eudat.models.data.files.ContentFile;
@ -34,6 +36,7 @@ import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@Component
public class GrantManager {
@ -122,7 +125,12 @@ public class GrantManager {
QueryableList<eu.eudat.data.entities.Grant> items = apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(grantCriteria.getCriteria());
QueryableList<eu.eudat.data.entities.Grant> authItems = apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getAuthenticated(items, userInfo);
List<eu.eudat.models.data.grant.Grant> grants = authItems.select(item -> new Grant().fromDataModel(item));
List<Map<String, String>> remoteRepos = remoteFetcher.getGrants(grantCriteria.getCriteria().getLike());
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(grantCriteria.getCriteria().getLike());
if (externalUrlCriteria.getPage() == null) externalUrlCriteria.setPage("0");
if (grantCriteria.getCriteria().getFunderReference() != null) externalUrlCriteria.setFunderId(grantCriteria.getCriteria().getFunderReference());
List<Map<String, String>> remoteRepos = remoteFetcher.getGrants(externalUrlCriteria);
GrantsExternalSourcesModel grantsExternalSourcesModel = new GrantsExternalSourcesModel().fromExternalItem(remoteRepos);
for (ExternalSourcesItemModel externalListingItem : grantsExternalSourcesModel) {
eu.eudat.models.data.grant.Grant grant = apiContext.getOperationsContext().getBuilderFactory().getBuilder(GrantBuilder.class)
@ -135,6 +143,7 @@ public class GrantManager {
grants.add(grant);
}
grants.sort(Comparator.comparing(Grant::getLabel));
grants = grants.stream().filter(ListHelper.distinctByKey(Grant::getLabel)).collect(Collectors.toList());
return grants;
}

7
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java
View File

@ -1,5 +1,7 @@
package eu.eudat.logic.managers;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.utilities.helpers.ListHelper;
import eu.eudat.models.data.external.ProjectsExternalSourcesModel;
import eu.eudat.models.data.project.Project;
import eu.eudat.data.query.items.item.project.ProjectCriteriaRequest;
@ -16,6 +18,7 @@ import org.springframework.stereotype.Component;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Component
public class ProjectManager {
@ -35,7 +38,8 @@ public class ProjectManager {
QueryableList<eu.eudat.data.entities.Project> items = apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(projectCriteria.getCriteria());
QueryableList<eu.eudat.data.entities.Project> authItems = apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getAuthenticated(items, userInfo);
List<Project> projects = authItems.select(item -> new Project().fromDataModel(item));
List<Map<String, String>> remoteRepos = remoteFetcher.getProjects(projectCriteria.getCriteria().getLike());
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(projectCriteria.getCriteria().getLike());
List<Map<String, String>> remoteRepos = remoteFetcher.getProjects(externalUrlCriteria);
ProjectsExternalSourcesModel projectsExternalSourcesModel = new ProjectsExternalSourcesModel().fromExternalItem(remoteRepos);
for (ExternalSourcesItemModel externalListingItem : projectsExternalSourcesModel) {
eu.eudat.models.data.project.Project project = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ProjectBuilder.class)
@ -48,6 +52,7 @@ public class ProjectManager {
projects.add(project);
}
projects.sort(Comparator.comparing(Project::getLabel));
projects = projects.stream().filter(ListHelper.distinctByKey(Project::getLabel)).collect(Collectors.toList());
return projects;
}
}

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/RegistryManager.java
View File

@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.dao.criteria.RegistryCriteria;
import eu.eudat.data.entities.Registry;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.services.ApiContext;
@ -33,7 +34,8 @@ public class RegistryManager {
}
public List<RegistryModel> getRegistries(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRegistries(query, type);
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRegistries(externalUrlCriteria, type);
RegistryCriteria criteria = new RegistryCriteria();
if (!query.isEmpty()) criteria.setLike(query);

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ResearcherManager.java
View File

@ -2,6 +2,7 @@ package eu.eudat.logic.managers;
import eu.eudat.logic.builders.model.models.ResearcherBuilder;
import eu.eudat.data.entities.Researcher;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.models.data.external.ExternalSourcesItemModel;
import eu.eudat.models.data.external.ResearchersExternalSourcesModel;
import eu.eudat.data.query.items.item.researcher.ResearcherCriteriaRequest;
@ -45,7 +46,8 @@ public class ResearcherManager {
QueryableList<eu.eudat.data.entities.Researcher> items = apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(researcherCriteriaRequest.getCriteria());
items.where((builder, root) -> builder.equal(root.get("creationUser").get("id"), principal.getId()));
List<eu.eudat.models.data.dmp.Researcher> researchers = items.select(item -> new eu.eudat.models.data.dmp.Researcher().fromDataModel(item));
List<Map<String, String>> remoteRepos = remoteFetcher.getResearchers(researcherCriteriaRequest.getCriteria().getName(),null);
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(researcherCriteriaRequest.getCriteria().getName());
List<Map<String, String>> remoteRepos = remoteFetcher.getResearchers(externalUrlCriteria,null);
ResearchersExternalSourcesModel researchersExternalSourcesModel = new ResearchersExternalSourcesModel().fromExternalItem(remoteRepos);
for (ExternalSourcesItemModel externalListingItem : researchersExternalSourcesModel) {
eu.eudat.models.data.dmp.Researcher researcher = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ResearcherBuilder.class)

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ServiceManager.java
View File

@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.dao.criteria.ServiceCriteria;
import eu.eudat.data.entities.Service;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.services.ApiContext;
@ -36,7 +37,8 @@ public class ServiceManager {
}
public List<ServiceModel> getServices(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getServices(query, type);
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getServices(externalUrlCriteria, type);
ServiceCriteria criteria = new ServiceCriteria();
if (!query.isEmpty()) criteria.setLike(query);

43
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/ExternalUrlCriteria.java Normal file
View File

@ -0,0 +1,43 @@
package eu.eudat.logic.proxy.config;
public class ExternalUrlCriteria {
private String like;
private String page;
private String pageSize;
private String funderId;
public String getLike() {
return like;
}
public void setLike(String like) {
this.like = like;
}
public String getPage() {
return page;
}
public void setPage(String page) {
this.page = page;
}
public String getPageSize() {
return pageSize;
}
public void setPageSize(String pageSize) {
this.pageSize = pageSize;
}
public String getFunderId() {
return funderId;
}
public void setFunderId(String funderId) {
this.funderId = funderId;
}
public ExternalUrlCriteria(String like) {
this.like = like;
}
public ExternalUrlCriteria() {
}
}

37
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java
View File

@ -8,18 +8,16 @@ public class UrlConfiguration {
private String key;
private String label;
private Integer ordinal;
private DataSearchConfiguration search;
private DataPageConfiguration page;
private String url;
private DataUrlConfiguration data;
private String type;
private String paginationPath;
private String contentType;
private String funderQuery;
public String getKey() {
return key;
}
@XmlElement(name = "key")
public void setKey(String key) {
this.key = key;
@ -28,7 +26,6 @@ public class UrlConfiguration {
public String getLabel() {
return label;
}
@XmlElement(name = "label")
public void setLabel(String label) {
this.label = label;
@ -37,7 +34,6 @@ public class UrlConfiguration {
public String getUrl() {
return url;
}
@XmlElement(name = "url")
public void setUrl(String url) {
this.url = url;
@ -46,34 +42,14 @@ public class UrlConfiguration {
public Integer getOrdinal() {
return ordinal;
}
@XmlElement(name = "ordinal")
public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal;
}
public DataSearchConfiguration getSearch() {
return search;
}
@XmlElement(name = "search")
public void setSearch(DataSearchConfiguration search) {
this.search = search;
}
public DataPageConfiguration getPage() {
return page;
}
@XmlElement(name = "page")
public void setPage(DataPageConfiguration page) {
this.page = page;
}
public DataUrlConfiguration getData() {
return data;
}
@XmlElement(name = "data")
public void setData(DataUrlConfiguration data) {
this.data = data;
@ -82,7 +58,6 @@ public class UrlConfiguration {
public String getPaginationPath() {
return paginationPath;
}
@XmlElement(name = "paginationpath")
public void setPaginationPath(String paginationPath) {
this.paginationPath = paginationPath;
@ -91,7 +66,6 @@ public class UrlConfiguration {
public String getType() {
return type;
}
@XmlElement(name = "type")
public void setType(String type) {
this.type = type;
@ -100,9 +74,16 @@ public class UrlConfiguration {
public String getContentType() {
return contentType;
}
@XmlElement(name = "contenttype")
public void setContentType(String contentType) {
this.contentType = contentType;
}
public String getFunderQuery() {
return funderQuery;
}
@XmlElement(name = "funderQuery")
public void setFunderQuery(String funderQuery) {
this.funderQuery = funderQuery;
}
}

120
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -15,9 +15,11 @@ import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.*;
import java.util.stream.Collectors;
@ -35,127 +37,157 @@ public class RemoteFetcher {
}
@Cacheable("repositories")
public List<Map<String, String>> getRepositories(String query, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getRepositories(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getRepositories().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("grants")
public List<Map<String, String>> getGrants(String query) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getGrants(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getGrants().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("projects")
public List<Map<String, String>> getProjects(String query) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getProjects(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("funders")
public List<Map<String, String>> getFunders(String query) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getFunders(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("organisations")
public List<Map<String, String>> getOrganisations(String query, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getOrganisations(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getOrganisations().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("registries")
public List<Map<String, String>> getRegistries(String query, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getRegistries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getRegistries().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("services")
public List<Map<String, String>> getServices(String query, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getServices(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getServices().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("researchers")
public List<Map<String, String>> getResearchers(String query, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getResearchers(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getResearchers().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("tags")
public List<Map<String, String>> getTags(String query, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getTags(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getTags().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getTags().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getTags().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("datasets")
public List<Map<String, String>> getDatasets(String query, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, String>> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getDatasets().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet {
private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
if (urlConfigs == null || urlConfigs.isEmpty())
throw new NoURLFound("No Repository urls found in configuration");
Collections.sort(urlConfigs, Comparator.comparing(UrlConfiguration::getOrdinal));
urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
List<Map<String, String>> results = new LinkedList<>();
for (UrlConfiguration urlConfig : urlConfigs) {
ifFunderQueryExist(urlConfig, externalUrlCriteria);
if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel(), urlConfig.getSearch(), urlConfig.getPage(), urlConfig.getContentType()));
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getContentType()));
} else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query));
results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
}
}
return results;
}
private void ifFunderQueryExist(UrlConfiguration urlConfiguration, ExternalUrlCriteria externalUrlCriteria) {
if (urlConfiguration.getFunderQuery() != null) {
if (externalUrlCriteria.getFunderId() != null) {
urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery()));
}
else {
urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", ""));
}
}
}
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key, DataSearchConfiguration search, DataPageConfiguration pageConfig, String contentType) throws HugeResultSet {
private String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria) {
String completedPath = path;
if (externalUrlCriteria.getLike() != null) {
if (path.contains("openaire") && externalUrlCriteria.getLike().equals(""))
completedPath = completedPath.replaceAll("\\{like}", "*");
else
completedPath = completedPath.replaceAll("\\{like}", externalUrlCriteria.getLike());
} else {
completedPath = completedPath.replace("{like}", "");
}
if (externalUrlCriteria.getFunderId() != null) {
String funderId = externalUrlCriteria.getFunderId();
try {
funderId = URLEncoder.encode(externalUrlCriteria.getFunderId(), "UTF-8");
} catch (UnsupportedEncodingException e) {
e.printStackTrace();
}
completedPath = completedPath.replace("{funderId}", funderId);
}
if (externalUrlCriteria.getPage() != null) {
completedPath = completedPath.replace("{page}", externalUrlCriteria.getPage());
} else {
completedPath = completedPath.replace("{page}", "1");
}
if (externalUrlCriteria.getPageSize() != null) {
completedPath = completedPath.replace("{pageSize}", externalUrlCriteria.getPageSize());
} else {
completedPath = completedPath.replace("{pageSize}", "10");
}
return completedPath;
}
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String key, String contentType) throws HugeResultSet {
Set<Integer> pages = new HashSet<>();
String searchQuery = "";
Results results = null;
if (search != null && search.getType() != null && !search.getType().trim().isEmpty() &&
search.getQueryParam() != null && !search.getQueryParam().trim().isEmpty() &&
query != null && !query.trim().isEmpty()) {
if (search.getType().equals("queryParam"))
searchQuery = "&" + search.getQueryParam() + "=" + query;
}
if (pageConfig != null) {
if (pageConfig.getType().equals("queryParam"))
results = getResultsFromUrl(path + "?" + pageConfig.getPageParam() + "=1" + searchQuery, jsonDataPath, jsonPaginationPath, contentType);
} else {
results = getResultsFromUrl(path, jsonDataPath, jsonPaginationPath, contentType);
}
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria);
Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType);
if (fetchStrategy == FetchStrategy.FIRST)
return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
@ -165,17 +197,15 @@ public class RemoteFetcher {
Long maxResults = configLoader.getExternalUrls().getMaxresults();
if ((maxResults > 0 && results != null) && (results.getPagination().get("count") > maxResults))
throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query");
String finalSearchQuery = searchQuery;
Optional<Results> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(path + "?page=" + page + finalSearchQuery, jsonDataPath, jsonPaginationPath, contentType))
.map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType))
.reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults());
return result1;
});
Results remainingResults = optionalResults.orElseGet(Results::new);
remainingResults.getResults().addAll(results.getResults());
return remainingResults.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
@ -203,7 +233,7 @@ public class RemoteFetcher {
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"),
jsonContext.read(jsonPaginationPath));
} else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openair.eu
} else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),

48
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ProjectsExternalSourcesModel.java vendored
View File

@ -1,19 +1,61 @@
package eu.eudat.models.data.external;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.io.IOException;
import java.util.List;
import java.util.Map;
public class ProjectsExternalSourcesModel extends ExternalListingItem<ProjectsExternalSourcesModel>{
public class ProjectsExternalSourcesModel extends ExternalListingItem<ProjectsExternalSourcesModel> {
private static final ObjectMapper mapper = new ObjectMapper();
@Override
public ProjectsExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setRemoteId(item.get("pid"));
model.setUri(item.get("uri"));
try {
JsonNode node = mapper.readTree(mapper.writeValueAsBytes(item));
JsonNode name = node.get("name");
if (name != null && !name.isNull() && name.isObject()) {
model.setName(node.get("name").get("$").asText());
} else {
model.setName(item.get("name"));
}
JsonNode pid = node.get("pid");
if (pid != null && !pid.isNull() && pid.isObject()) {
model.setRemoteId(node.get("pid").get("$").asText());
} else {
model.setRemoteId(item.get("pid"));
}
JsonNode uri = node.get("uri");
if (uri != null && !uri.isNull() && uri.isObject()) {
model.setUri(node.get("uri").get("$").asText());
} else {
model.setUri(item.get("uri"));
}
JsonNode description = node.get("description");
if (description != null && !description.isNull() && description.isObject()) {
model.setDescription(node.get("description").get("$").asText());
} else {
model.setDescription(item.get("description"));
}
JsonNode tag = node.get("tag");
if (tag != null && !tag.isNull() && tag.isObject()) {
model.setTag(node.get("tag").get("$").asText());
} else {
model.setTag(item.get("tag"));
}
this.add(model);
} catch (IOException e) {
e.printStackTrace();
}
}
return this;
}

98
dmp-backend/web/src/main/resources/ExternalUrls.xml
View File

@ -128,7 +128,8 @@
<!-- <key>cristin</key>-->
<!-- <label>Cristin</label>-->
<!-- <ordinal>1</ordinal>-->
<!-- <url>https://eestore.paas2.uninett.no/api/projectrepo/</url>-->
<!-- <url>https://eestore.paas2.uninett.no/api/projectrepo/?search={like}&amp;page={page}&amp;size={pageSize}</url>-->
<!-- <contenttype>application/vnd.api+json; charset=utf-8</contenttype>-->
<!-- <data>-->
<!-- <path>$['data'][*]['attributes']</path>-->
<!-- <fields>-->
@ -140,6 +141,25 @@
<!-- </data>-->
<!-- <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
<!-- </urlConfig>-->
<urlConfig>
<key>openAIRE</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<url>https://services.openaire.eu/search/v2/api/resources?query=((oaftype%20exact%20project)%20and%20((projectcode_nt%20exact%20%22{like}%22)or({like}))){funderQuery}&amp;page={page}&amp;size={pageSize}&amp;format=json</url>
<funderQuery>&amp;fq=(funder%20exact%20%22{funderId}%22)</funderQuery>
<contenttype>application/json; charset=utf-8</contenttype>
<data>
<path>$['results'][*]['result']['metadata']['oaf:entity']['oaf:project']</path>
<fields>
<id>'originalId'</id>
<name>'title'</name>
<count>'count'</count>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
<!-- <urlConfig>-->
<!-- <key>openAire</key>-->
<!-- <label>OpenAIRE</label>-->
@ -157,37 +177,12 @@
<!-- </data>-->
<!-- <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
<!-- </urlConfig>-->
<urlConfig>
<key>internal</key>
<label>Internal</label>
<ordinal>1</ordinal>
<url>GrantInternalMockUpData.json</url>
<type>Internal</type>
<data>
<path>$['data'][*]['attributes']</path>
<fields>
<id>'pid'</id>
<name>'name'</name>
<uri>'uri'</uri>
<description>'description'</description>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
</urls>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</grants>
<projects>
<urls>
<!-- <urlConfig>-->
<!-- <key>cristin</key>-->
<!-- <label>Cristin</label>-->
<!-- <key>internal</key>-->
<!-- <label>Internal</label>-->
<!-- <ordinal>1</ordinal>-->
<!-- <type>External</type>-->
<!-- <url>https://eestore.paas2.uninett.no/api/projectrepo/</url>-->
<!-- <url>GrantInternalMockUpData.json</url>-->
<!-- <type>Internal</type>-->
<!-- <data>-->
<!-- <path>$['data'][*]['attributes']</path>-->
<!-- <fields>-->
@ -199,6 +194,49 @@
<!-- </data>-->
<!-- <paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>-->
<!-- </urlConfig>-->
</urls>
<fetchMode>FIRST</fetchMode> <!-- EITHER 'FIRST' OR 'ALL' -->
</grants>
<projects>
<urls>
<urlConfig>
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
<contenttype>application/vnd.api+json; charset=utf-8</contenttype>
<data>
<path>$['data'][*]['attributes']</path>
<fields>
<id>'pid'</id>
<name>'name'</name>
<uri>'uri'</uri>
<description>'description'</description>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
<urlConfig>
<key>openAIRE</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://services.openaire.eu/search/v2/api/resources?query=((oaftype%20exact%20project)%20and%20((projectcode_nt%20exact%20%22*%22)or(*)))&amp;fq=(funder%20exact%20{funderId})&amp;page={page}&amp;size={pageSize}&amp;format=json</url>
<contenttype>application/json; charset=utf-8</contenttype>
<data>
<path>$['response']['results']['result'][*]['metadata']['oaf:entity']['oaf:project']</path>
<fields>
<id>'originalId'</id>
<name>'title'</name>
<count>'count'</count>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
<!-- <urlConfig>-->
<!-- <key>openAire</key>-->
<!-- <label>OpenAIRE</label>-->