From dc7f05ffd2a13c4e211002483d6a2a40264b60ef Mon Sep 17 00:00:00 2001 From: gkolokythas Date: Thu, 3 Oct 2019 14:06:44 +0300 Subject: [PATCH] Refactors external url fetching to support more complex APIs and adds requested external url on "Grant". (Issue #179) --- .../eudat/data/dao/entities/GrantDaoImpl.java | 3 + .../eu/eudat/controllers/Organisations.java | 4 +- .../eu/eudat/controllers/TagController.java | 4 +- .../managers/DataManagementPlanManager.java | 1 - .../logic/managers/DataRepositoryManager.java | 4 +- .../eudat/logic/managers/FunderManager.java | 4 +- .../eu/eudat/logic/managers/GrantManager.java | 11 +- .../eudat/logic/managers/ProjectManager.java | 7 +- .../eudat/logic/managers/RegistryManager.java | 4 +- .../logic/managers/ResearcherManager.java | 4 +- .../eudat/logic/managers/ServiceManager.java | 4 +- .../proxy/config/ExternalUrlCriteria.java | 43 ++++++ .../logic/proxy/config/UrlConfiguration.java | 37 ++--- .../logic/proxy/fetching/RemoteFetcher.java | 128 +++++++++++------- .../ProjectsExternalSourcesModel.java | 56 +++++++- .../web/src/main/resources/ExternalUrls.xml | 98 ++++++++++---- 16 files changed, 288 insertions(+), 124 deletions(-) create mode 100644 dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/ExternalUrlCriteria.java diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/GrantDaoImpl.java b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/GrantDaoImpl.java index 47d22c064..bb91b39ad 100644 --- a/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/GrantDaoImpl.java +++ b/dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/GrantDaoImpl.java @@ -11,6 +11,7 @@ import eu.eudat.types.grant.GrantStateType; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; +import schemasMicrosoftComOfficeOffice.LeftDocument; import javax.persistence.criteria.JoinType; import java.util.Date; @@ -51,6 +52,8 @@ public class GrantDaoImpl extends DatabaseAccess implements GrantDao { } if (criteria.getFunderId() != null && !criteria.getFunderId().trim().isEmpty()) query.where((builder, root) -> builder.equal(root.get("funder").get("id"), UUID.fromString(criteria.getFunderId()))); + if (criteria.getFunderReference() != null && !criteria.getFunderReference().isEmpty()) + query.where((builder, root) -> builder.equal(root.join("funder", JoinType.LEFT).get("reference"), criteria.getFunderReference())); query.where((builder, root) -> builder.notEqual(root.get("status"), Grant.Status.DELETED.getValue())); return query; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/Organisations.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/Organisations.java index 8ba8d4ac2..76b5557f3 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/Organisations.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/Organisations.java @@ -2,6 +2,7 @@ package eu.eudat.controllers; import eu.eudat.data.query.items.table.organisations.OrganisationsTableRequest; import eu.eudat.logic.managers.OrganisationsManager; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import eu.eudat.logic.services.ApiContext; @@ -38,7 +39,8 @@ public class Organisations extends BaseController { ResponseEntity> listExternalOrganisations( @RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type ) throws HugeResultSet, NoURLFound { - List> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getOrganisations(query, type); + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); + List> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getOrganisations(externalUrlCriteria, type); OrganisationsExternalSourcesModel organisationsExternalSourcesModel = new OrganisationsExternalSourcesModel().fromExternalItem(remoteRepos); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(organisationsExternalSourcesModel).status(ApiMessageCode.NO_MESSAGE)); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/TagController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/TagController.java index 02bcb3186..752000cf9 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/TagController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/TagController.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.elastic.criteria.TagCriteria; import eu.eudat.elastic.entities.Dataset; import eu.eudat.elastic.repository.Repository; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import eu.eudat.logic.services.ApiContext; @@ -47,7 +48,8 @@ public class TagController extends BaseController { public @ResponseBody ResponseEntity> listExternalTagModel( @RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type) throws HugeResultSet, NoURLFound, IOException { - List> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(query, type); + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); + List> remoteRepos = this.getApiContext().getOperationsContext().getRemoteFetcher().getTags(externalUrlCriteria, type); TagExternalSourcesModel researchersExternalSourcesModel = new TagExternalSourcesModel().fromExternalItem(remoteRepos); // ObjectMapper mapper = new ObjectMapper(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java index 396e0008d..2e8d3a6c5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java @@ -659,7 +659,6 @@ public class DataManagementPlanManager { eu.eudat.data.entities.Funder funderEntity = funderDao.getWithCritetia(criteria).getSingleOrDefault(); if (funderEntity != null) funder.setId(funderEntity.getId()); else { - // funder.setType(Funder.FunderType.EXTERNAL.getValue()); funderDao.createOrUpdate(funder); } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataRepositoryManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataRepositoryManager.java index 852d12a63..e7143d223 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataRepositoryManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataRepositoryManager.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.data.dao.criteria.DataRepositoryCriteria; import eu.eudat.data.entities.DataRepository; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import eu.eudat.logic.services.ApiContext; @@ -35,7 +36,8 @@ public class DataRepositoryManager { } public List getDataRepositories(String query, String type, Principal principal) throws HugeResultSet, NoURLFound { - List> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRepositories(query, type); + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); + List> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRepositories(externalUrlCriteria, type); DataRepositoryCriteria criteria = new DataRepositoryCriteria(); if (!query.isEmpty()) criteria.setLike(query); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java index 640746082..8bdfacaab 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java @@ -2,6 +2,7 @@ package eu.eudat.logic.managers; import eu.eudat.data.query.items.item.funder.FunderCriteriaRequest; import eu.eudat.logic.builders.model.models.FunderBuilder; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import eu.eudat.logic.proxy.fetching.RemoteFetcher; @@ -37,7 +38,8 @@ public class FunderManager { QueryableList items = apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(funderCriteria.getCriteria()); QueryableList authItems = apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getAuthenticated(items, userInfo); List funders = authItems.select(item -> new eu.eudat.models.data.funder.Funder().fromDataModel(item)); - List> remoteRepos = remoteFetcher.getFunders(funderCriteria.getCriteria().getLike()); + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(funderCriteria.getCriteria().getLike()); + List> remoteRepos = remoteFetcher.getFunders(externalUrlCriteria); FundersExternalSourcesModel fundersExternalSourcesModel = new FundersExternalSourcesModel().fromExternalItem(remoteRepos); for (ExternalSourcesItemModel externalListingItem : fundersExternalSourcesModel) { eu.eudat.models.data.funder.Funder funder = apiContext.getOperationsContext().getBuilderFactory().getBuilder(FunderBuilder.class) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java index 6e3e07d1d..d1eb90c1f 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java @@ -10,7 +10,9 @@ import eu.eudat.data.dao.entities.GrantDao; import eu.eudat.data.entities.Content; import eu.eudat.data.entities.DMP; import eu.eudat.exceptions.files.TempFileNotFoundException; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.services.operations.DatabaseRepository; +import eu.eudat.logic.utilities.helpers.ListHelper; import eu.eudat.models.data.external.ExternalSourcesItemModel; import eu.eudat.models.data.external.GrantsExternalSourcesModel; import eu.eudat.models.data.files.ContentFile; @@ -34,6 +36,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; @Component public class GrantManager { @@ -122,7 +125,12 @@ public class GrantManager { QueryableList items = apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(grantCriteria.getCriteria()); QueryableList authItems = apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getAuthenticated(items, userInfo); List grants = authItems.select(item -> new Grant().fromDataModel(item)); - List> remoteRepos = remoteFetcher.getGrants(grantCriteria.getCriteria().getLike()); + + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(grantCriteria.getCriteria().getLike()); + if (externalUrlCriteria.getPage() == null) externalUrlCriteria.setPage("0"); + if (grantCriteria.getCriteria().getFunderReference() != null) externalUrlCriteria.setFunderId(grantCriteria.getCriteria().getFunderReference()); + List> remoteRepos = remoteFetcher.getGrants(externalUrlCriteria); + GrantsExternalSourcesModel grantsExternalSourcesModel = new GrantsExternalSourcesModel().fromExternalItem(remoteRepos); for (ExternalSourcesItemModel externalListingItem : grantsExternalSourcesModel) { eu.eudat.models.data.grant.Grant grant = apiContext.getOperationsContext().getBuilderFactory().getBuilder(GrantBuilder.class) @@ -135,6 +143,7 @@ public class GrantManager { grants.add(grant); } grants.sort(Comparator.comparing(Grant::getLabel)); + grants = grants.stream().filter(ListHelper.distinctByKey(Grant::getLabel)).collect(Collectors.toList()); return grants; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java index 258e5ef2d..fd7bff87b 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java @@ -1,5 +1,7 @@ package eu.eudat.logic.managers; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; +import eu.eudat.logic.utilities.helpers.ListHelper; import eu.eudat.models.data.external.ProjectsExternalSourcesModel; import eu.eudat.models.data.project.Project; import eu.eudat.data.query.items.item.project.ProjectCriteriaRequest; @@ -16,6 +18,7 @@ import org.springframework.stereotype.Component; import java.util.Comparator; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Component public class ProjectManager { @@ -35,7 +38,8 @@ public class ProjectManager { QueryableList items = apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(projectCriteria.getCriteria()); QueryableList authItems = apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getAuthenticated(items, userInfo); List projects = authItems.select(item -> new Project().fromDataModel(item)); - List> remoteRepos = remoteFetcher.getProjects(projectCriteria.getCriteria().getLike()); + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(projectCriteria.getCriteria().getLike()); + List> remoteRepos = remoteFetcher.getProjects(externalUrlCriteria); ProjectsExternalSourcesModel projectsExternalSourcesModel = new ProjectsExternalSourcesModel().fromExternalItem(remoteRepos); for (ExternalSourcesItemModel externalListingItem : projectsExternalSourcesModel) { eu.eudat.models.data.project.Project project = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ProjectBuilder.class) @@ -48,6 +52,7 @@ public class ProjectManager { projects.add(project); } projects.sort(Comparator.comparing(Project::getLabel)); + projects = projects.stream().filter(ListHelper.distinctByKey(Project::getLabel)).collect(Collectors.toList()); return projects; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/RegistryManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/RegistryManager.java index 5fe697bbc..8cbbfd52d 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/RegistryManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/RegistryManager.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.data.dao.criteria.RegistryCriteria; import eu.eudat.data.entities.Registry; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import eu.eudat.logic.services.ApiContext; @@ -33,7 +34,8 @@ public class RegistryManager { } public List getRegistries(String query, String type, Principal principal) throws HugeResultSet, NoURLFound { - List> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRegistries(query, type); + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); + List> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRegistries(externalUrlCriteria, type); RegistryCriteria criteria = new RegistryCriteria(); if (!query.isEmpty()) criteria.setLike(query); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ResearcherManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ResearcherManager.java index 950621c8b..e09988e80 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ResearcherManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ResearcherManager.java @@ -2,6 +2,7 @@ package eu.eudat.logic.managers; import eu.eudat.logic.builders.model.models.ResearcherBuilder; import eu.eudat.data.entities.Researcher; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.models.data.external.ExternalSourcesItemModel; import eu.eudat.models.data.external.ResearchersExternalSourcesModel; import eu.eudat.data.query.items.item.researcher.ResearcherCriteriaRequest; @@ -45,7 +46,8 @@ public class ResearcherManager { QueryableList items = apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(researcherCriteriaRequest.getCriteria()); items.where((builder, root) -> builder.equal(root.get("creationUser").get("id"), principal.getId())); List researchers = items.select(item -> new eu.eudat.models.data.dmp.Researcher().fromDataModel(item)); - List> remoteRepos = remoteFetcher.getResearchers(researcherCriteriaRequest.getCriteria().getName(),null); + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(researcherCriteriaRequest.getCriteria().getName()); + List> remoteRepos = remoteFetcher.getResearchers(externalUrlCriteria,null); ResearchersExternalSourcesModel researchersExternalSourcesModel = new ResearchersExternalSourcesModel().fromExternalItem(remoteRepos); for (ExternalSourcesItemModel externalListingItem : researchersExternalSourcesModel) { eu.eudat.models.data.dmp.Researcher researcher = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ResearcherBuilder.class) diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ServiceManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ServiceManager.java index 503bfb007..dbdf38c36 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ServiceManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ServiceManager.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.data.dao.criteria.ServiceCriteria; import eu.eudat.data.entities.Service; +import eu.eudat.logic.proxy.config.ExternalUrlCriteria; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import eu.eudat.logic.services.ApiContext; @@ -36,7 +37,8 @@ public class ServiceManager { } public List getServices(String query, String type, Principal principal) throws HugeResultSet, NoURLFound { - List> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getServices(query, type); + ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); + List> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getServices(externalUrlCriteria, type); ServiceCriteria criteria = new ServiceCriteria(); if (!query.isEmpty()) criteria.setLike(query); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/ExternalUrlCriteria.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/ExternalUrlCriteria.java new file mode 100644 index 000000000..fd0ea091c --- /dev/null +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/ExternalUrlCriteria.java @@ -0,0 +1,43 @@ +package eu.eudat.logic.proxy.config; + +public class ExternalUrlCriteria { + private String like; + private String page; + private String pageSize; + private String funderId; + + public String getLike() { + return like; + } + public void setLike(String like) { + this.like = like; + } + + public String getPage() { + return page; + } + public void setPage(String page) { + this.page = page; + } + + public String getPageSize() { + return pageSize; + } + public void setPageSize(String pageSize) { + this.pageSize = pageSize; + } + + public String getFunderId() { + return funderId; + } + public void setFunderId(String funderId) { + this.funderId = funderId; + } + + public ExternalUrlCriteria(String like) { + this.like = like; + } + + public ExternalUrlCriteria() { + } +} diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java index 3a14277e1..4d19000eb 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java @@ -8,18 +8,16 @@ public class UrlConfiguration { private String key; private String label; private Integer ordinal; - private DataSearchConfiguration search; - private DataPageConfiguration page; private String url; private DataUrlConfiguration data; private String type; private String paginationPath; private String contentType; + private String funderQuery; public String getKey() { return key; } - @XmlElement(name = "key") public void setKey(String key) { this.key = key; @@ -28,7 +26,6 @@ public class UrlConfiguration { public String getLabel() { return label; } - @XmlElement(name = "label") public void setLabel(String label) { this.label = label; @@ -37,7 +34,6 @@ public class UrlConfiguration { public String getUrl() { return url; } - @XmlElement(name = "url") public void setUrl(String url) { this.url = url; @@ -46,34 +42,14 @@ public class UrlConfiguration { public Integer getOrdinal() { return ordinal; } - @XmlElement(name = "ordinal") public void setOrdinal(Integer ordinal) { this.ordinal = ordinal; } - public DataSearchConfiguration getSearch() { - return search; - } - - @XmlElement(name = "search") - public void setSearch(DataSearchConfiguration search) { - this.search = search; - } - - public DataPageConfiguration getPage() { - return page; - } - - @XmlElement(name = "page") - public void setPage(DataPageConfiguration page) { - this.page = page; - } - public DataUrlConfiguration getData() { return data; } - @XmlElement(name = "data") public void setData(DataUrlConfiguration data) { this.data = data; @@ -82,7 +58,6 @@ public class UrlConfiguration { public String getPaginationPath() { return paginationPath; } - @XmlElement(name = "paginationpath") public void setPaginationPath(String paginationPath) { this.paginationPath = paginationPath; @@ -91,7 +66,6 @@ public class UrlConfiguration { public String getType() { return type; } - @XmlElement(name = "type") public void setType(String type) { this.type = type; @@ -100,9 +74,16 @@ public class UrlConfiguration { public String getContentType() { return contentType; } - @XmlElement(name = "contenttype") public void setContentType(String contentType) { this.contentType = contentType; } + + public String getFunderQuery() { + return funderQuery; + } + @XmlElement(name = "funderQuery") + public void setFunderQuery(String funderQuery) { + this.funderQuery = funderQuery; + } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index 2af700653..31fba06e1 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -15,9 +15,11 @@ import org.springframework.stereotype.Service; import java.io.File; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.HttpURLConnection; import java.net.MalformedURLException; import java.net.URL; +import java.net.URLEncoder; import java.util.*; import java.util.stream.Collectors; @@ -35,127 +37,157 @@ public class RemoteFetcher { } @Cacheable("repositories") - public List> getRepositories(String query, String key) throws NoURLFound, HugeResultSet { + public List> getRepositories(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getRepositories().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRepositories().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("grants") - public List> getGrants(String query) throws NoURLFound, HugeResultSet { + public List> getGrants(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getGrants().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("projects") - public List> getProjects(String query) throws NoURLFound, HugeResultSet { + public List> getProjects(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getProjects().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("funders") - public List> getFunders(String query) throws NoURLFound, HugeResultSet { + public List> getFunders(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet { List urlConfigs = configLoader.getExternalUrls().getFunders().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("organisations") - public List> getOrganisations(String query, String key) throws NoURLFound, HugeResultSet { + public List> getOrganisations(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getOrganisations().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getOrganisations().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("registries") - public List> getRegistries(String query, String key) throws NoURLFound, HugeResultSet { + public List> getRegistries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getRegistries().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getRegistries().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("services") - public List> getServices(String query, String key) throws NoURLFound, HugeResultSet { + public List> getServices(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getServices().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getServices().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("researchers") - public List> getResearchers(String query, String key) throws NoURLFound, HugeResultSet { + public List> getResearchers(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getResearchers().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getResearchers().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("tags") - public List> getTags(String query, String key) throws NoURLFound, HugeResultSet { + public List> getTags(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getTags().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getTags().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getTags().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } @Cacheable("datasets") - public List> getDatasets(String query, String key) throws NoURLFound, HugeResultSet { + public List> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet { List urlConfigs = key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) : configLoader.getExternalUrls().getDatasets().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getDatasets().getFetchMode(); - return getAll(urlConfigs, fetchStrategy, query); + return getAll(urlConfigs, fetchStrategy, externalUrlCriteria); } - private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, String query) throws NoURLFound, HugeResultSet { + private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet { if (urlConfigs == null || urlConfigs.isEmpty()) throw new NoURLFound("No Repository urls found in configuration"); - Collections.sort(urlConfigs, Comparator.comparing(UrlConfiguration::getOrdinal)); + urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal)); List> results = new LinkedList<>(); for (UrlConfiguration urlConfig : urlConfigs) { + ifFunderQueryExist(urlConfig, externalUrlCriteria); if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { - results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel(), urlConfig.getSearch(), urlConfig.getPage(), urlConfig.getContentType())); + results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getContentType())); } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { - results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query)); + results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike())); } } return results; } + private void ifFunderQueryExist(UrlConfiguration urlConfiguration, ExternalUrlCriteria externalUrlCriteria) { + if (urlConfiguration.getFunderQuery() != null) { + if (externalUrlCriteria.getFunderId() != null) { + urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery())); + } + else { + urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", "")); + } + } + } - private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, String query, String key, DataSearchConfiguration search, DataPageConfiguration pageConfig, String contentType) throws HugeResultSet { + private String replaceCriteriaOnUrl(String path, ExternalUrlCriteria externalUrlCriteria) { + String completedPath = path; + if (externalUrlCriteria.getLike() != null) { + if (path.contains("openaire") && externalUrlCriteria.getLike().equals("")) + completedPath = completedPath.replaceAll("\\{like}", "*"); + else + completedPath = completedPath.replaceAll("\\{like}", externalUrlCriteria.getLike()); + } else { + completedPath = completedPath.replace("{like}", ""); + } + if (externalUrlCriteria.getFunderId() != null) { + String funderId = externalUrlCriteria.getFunderId(); + try { + funderId = URLEncoder.encode(externalUrlCriteria.getFunderId(), "UTF-8"); + } catch (UnsupportedEncodingException e) { + e.printStackTrace(); + } + completedPath = completedPath.replace("{funderId}", funderId); + } + if (externalUrlCriteria.getPage() != null) { + completedPath = completedPath.replace("{page}", externalUrlCriteria.getPage()); + } else { + completedPath = completedPath.replace("{page}", "1"); + } + if (externalUrlCriteria.getPageSize() != null) { + completedPath = completedPath.replace("{pageSize}", externalUrlCriteria.getPageSize()); + } else { + completedPath = completedPath.replace("{pageSize}", "10"); + } + return completedPath; + } + + private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String key, String contentType) throws HugeResultSet { Set pages = new HashSet<>(); - String searchQuery = ""; - Results results = null; - if (search != null && search.getType() != null && !search.getType().trim().isEmpty() && - search.getQueryParam() != null && !search.getQueryParam().trim().isEmpty() && - query != null && !query.trim().isEmpty()) { - if (search.getType().equals("queryParam")) - searchQuery = "&" + search.getQueryParam() + "=" + query; - } - - if (pageConfig != null) { - if (pageConfig.getType().equals("queryParam")) - results = getResultsFromUrl(path + "?" + pageConfig.getPageParam() + "=1" + searchQuery, jsonDataPath, jsonPaginationPath, contentType); - } else { - results = getResultsFromUrl(path, jsonDataPath, jsonPaginationPath, contentType); - } + String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria); + Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType); if (fetchStrategy == FetchStrategy.FIRST) return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList()); @@ -165,17 +197,15 @@ public class RemoteFetcher { Long maxResults = configLoader.getExternalUrls().getMaxresults(); if ((maxResults > 0 && results != null) && (results.getPagination().get("count") > maxResults)) - throw new HugeResultSet("The submitted search query " + query + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); + throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); - String finalSearchQuery = searchQuery; Optional optionalResults = pages.parallelStream() - .map(page -> getResultsFromUrl(path + "?page=" + page + finalSearchQuery, jsonDataPath, jsonPaginationPath, contentType)) + .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType)) .reduce((result1, result2) -> { result1.getResults().addAll(result2.getResults()); return result1; }); Results remainingResults = optionalResults.orElseGet(Results::new); - remainingResults.getResults().addAll(results.getResults()); return remainingResults.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList()); @@ -203,7 +233,7 @@ public class RemoteFetcher { + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getSource() + "]"), jsonContext.read(jsonPaginationPath)); - } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openair.eu + } else if (jsonDataPath.getFieldsUrlConfiguration().getCount() != null) { // parsing services.openaire.eu results = new Results(jsonContext.read(jsonDataPath.getPath() + "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"), @@ -262,12 +292,12 @@ public class RemoteFetcher { } private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) { - if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) return "pid"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description"; - if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri"; - if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name"; if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source"; - if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count"; + if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count"; return null; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/external/ProjectsExternalSourcesModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/external/ProjectsExternalSourcesModel.java index 6f8402ee9..84537d5b2 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/external/ProjectsExternalSourcesModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/external/ProjectsExternalSourcesModel.java @@ -1,19 +1,61 @@ package eu.eudat.models.data.external; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import java.io.IOException; import java.util.List; import java.util.Map; -public class ProjectsExternalSourcesModel extends ExternalListingItem{ +public class ProjectsExternalSourcesModel extends ExternalListingItem { + private static final ObjectMapper mapper = new ObjectMapper(); + @Override public ProjectsExternalSourcesModel fromExternalItem(List> values) { for (Map item : values) { ExternalSourcesItemModel model = new ExternalSourcesItemModel(); - model.setRemoteId(item.get("pid")); - model.setUri(item.get("uri")); - model.setName(item.get("name")); - model.setDescription(item.get("description")); - model.setTag(item.get("tag")); - this.add(model); + try { + JsonNode node = mapper.readTree(mapper.writeValueAsBytes(item)); + JsonNode name = node.get("name"); + if (name != null && !name.isNull() && name.isObject()) { + model.setName(node.get("name").get("$").asText()); + } else { + model.setName(item.get("name")); + } + + JsonNode pid = node.get("pid"); + if (pid != null && !pid.isNull() && pid.isObject()) { + model.setRemoteId(node.get("pid").get("$").asText()); + } else { + model.setRemoteId(item.get("pid")); + } + + JsonNode uri = node.get("uri"); + if (uri != null && !uri.isNull() && uri.isObject()) { + model.setUri(node.get("uri").get("$").asText()); + } else { + model.setUri(item.get("uri")); + } + + JsonNode description = node.get("description"); + if (description != null && !description.isNull() && description.isObject()) { + model.setDescription(node.get("description").get("$").asText()); + } else { + model.setDescription(item.get("description")); + } + + JsonNode tag = node.get("tag"); + if (tag != null && !tag.isNull() && tag.isObject()) { + model.setTag(node.get("tag").get("$").asText()); + } else { + model.setTag(item.get("tag")); + } + + this.add(model); + + } catch (IOException e) { + e.printStackTrace(); + } } return this; } diff --git a/dmp-backend/web/src/main/resources/ExternalUrls.xml b/dmp-backend/web/src/main/resources/ExternalUrls.xml index ecadfe4a3..2af67ec07 100644 --- a/dmp-backend/web/src/main/resources/ExternalUrls.xml +++ b/dmp-backend/web/src/main/resources/ExternalUrls.xml @@ -128,7 +128,8 @@ - + + @@ -140,6 +141,25 @@ + + + openAIRE + + 1 + https://services.openaire.eu/search/v2/api/resources?query=((oaftype%20exact%20project)%20and%20((projectcode_nt%20exact%20%22{like}%22)or({like}))){funderQuery}&page={page}&size={pageSize}&format=json + &fq=(funder%20exact%20%22{funderId}%22) + application/json; charset=utf-8 + + $['results'][*]['result']['metadata']['oaf:entity']['oaf:project'] + + 'originalId' + 'title' + 'count' + + + $['meta']['pagination']['page','pages','count'] + + @@ -157,37 +177,12 @@ - - internal - - 1 - GrantInternalMockUpData.json - Internal - - $['data'][*]['attributes'] - - 'pid' - 'name' - 'uri' - 'description' - - - $['meta']['pagination']['page','pages','count'] - - - - - FIRST - - - - - - + + - - + + @@ -199,6 +194,49 @@ + + + + FIRST + + + + + + cristin + + 1 + External + https://eestore.paas2.uninett.no/api/projectrepo/ + application/vnd.api+json; charset=utf-8 + + $['data'][*]['attributes'] + + 'pid' + 'name' + 'uri' + 'description' + + + $['meta']['pagination']['page','pages','count'] + + + openAIRE + + 1 + External + https://services.openaire.eu/search/v2/api/resources?query=((oaftype%20exact%20project)%20and%20((projectcode_nt%20exact%20%22*%22)or(*)))&fq=(funder%20exact%20{funderId})&page={page}&size={pageSize}&format=json + application/json; charset=utf-8 + + $['response']['results']['result'][*]['metadata']['oaf:entity']['oaf:project'] + + 'originalId' + 'title' + 'count' + + + $['meta']['pagination']['page','pages','count'] +