From 9a90e3f8af0abd9bc99d21ddf2214e599b523909 Mon Sep 17 00:00:00 2001 From: gkolokythas Date: Thu, 5 Sep 2019 17:44:25 +0300 Subject: [PATCH] Implements the functionality of being able to add static data when using external services, and refactors Funder, Grant and Project to be configured by "ExternalUrs". (Issue #147) --- .../dynamicfunder/entities/MainProperty.java | 6 +- .../dynamicgrant/entities/MainProperty.java | 6 +- .../dynamicproject/entities/MainProperty.java | 6 +- .../eudat/logic/managers/FunderManager.java | 4 +- .../eu/eudat/logic/managers/GrantManager.java | 4 +- .../eudat/logic/managers/ProjectManager.java | 4 +- .../logic/proxy/config/UrlConfiguration.java | 9 ++ .../logic/proxy/fetching/RemoteFetcher.java | 68 ++++++++----- .../external/FundersExternalSourcesModel.java | 2 +- .../external/GrantsExternalSourcesModel.java | 2 +- .../ProjectsExternalSourcesModel.java | 2 +- .../web/src/main/resources/ExternalUrls.xml | 19 ++++ .../main/resources/FunderConfiguration.xml | 99 ------------------- .../main/resources/ProjectConfiguration.xml | 99 ------------------- .../resources/application-staging.properties | 4 +- 15 files changed, 93 insertions(+), 241 deletions(-) delete mode 100644 dmp-backend/web/src/main/resources/FunderConfiguration.xml delete mode 100644 dmp-backend/web/src/main/resources/ProjectConfiguration.xml diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicfunder/entities/MainProperty.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicfunder/entities/MainProperty.java index 1ff185ac5..997dc4aff 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicfunder/entities/MainProperty.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicfunder/entities/MainProperty.java @@ -11,7 +11,7 @@ public class MainProperty { private String name; private String queryProperty; private String externalFieldId; - private UrlConfiguration urlConfig; + private List urlConfig; private String externalFieldLabel; private List dependencies; private Boolean required; @@ -82,12 +82,12 @@ public class MainProperty { this.queryProperty = queryProperty; } - public UrlConfiguration getUrlConfig() { + public List getUrlConfig() { return urlConfig; } @XmlElement(name = "urlConfig") - public void setUrlConfig(UrlConfiguration urlConfig) { + public void setUrlConfig(List urlConfig) { this.urlConfig = urlConfig; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicgrant/entities/MainProperty.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicgrant/entities/MainProperty.java index ed4615a74..fd0b8732f 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicgrant/entities/MainProperty.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicgrant/entities/MainProperty.java @@ -14,7 +14,7 @@ public class MainProperty { private String name; private String queryProperty; private String externalFieldId; - private UrlConfiguration urlConfig; + private List urlConfig; private String externalFieldLabel; private List dependencies; private Boolean required; @@ -85,12 +85,12 @@ public class MainProperty { this.queryProperty = queryProperty; } - public UrlConfiguration getUrlConfig() { + public List getUrlConfig() { return urlConfig; } @XmlElement(name = "urlConfig") - public void setUrlConfig(UrlConfiguration urlConfig) { + public void setUrlConfig(List urlConfig) { this.urlConfig = urlConfig; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicproject/entities/MainProperty.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicproject/entities/MainProperty.java index 3eab4f3b5..9506b4d71 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicproject/entities/MainProperty.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/dynamicproject/entities/MainProperty.java @@ -11,7 +11,7 @@ public class MainProperty { private String name; private String queryProperty; private String externalFieldId; - private UrlConfiguration urlConfig; + private List urlConfig; private String externalFieldLabel; private List dependencies; private Boolean required; @@ -82,12 +82,12 @@ public class MainProperty { this.queryProperty = queryProperty; } - public UrlConfiguration getUrlConfig() { + public List getUrlConfig() { return urlConfig; } @XmlElement(name = "urlConfig") - public void setUrlConfig(UrlConfiguration urlConfig) { + public void setUrlConfig(List urlConfig) { this.urlConfig = urlConfig; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java index 1f2d732ca..8c882953a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java @@ -41,12 +41,12 @@ public class FunderManager { eu.eudat.models.data.funder.Funder funder = apiContext.getOperationsContext().getBuilderFactory().getBuilder(FunderBuilder.class) .reference(externalListingItem.getRemoteId()).label(externalListingItem.getName()) .status(eu.eudat.data.entities.Funder.Status.fromInteger(0)) - .source(externalListingItem.getSource()) + .source(externalListingItem.getTag()) .build(); funders.add(funder); } - funders.sort(Comparator.comparing(x -> x.getLabel())); + funders.sort(Comparator.comparing(Funder::getLabel)); return funders; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java index 6c4fae826..6e3e07d1d 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java @@ -129,12 +129,12 @@ public class GrantManager { .reference(externalListingItem.getRemoteId()).label(externalListingItem.getName()) .description(externalListingItem.getDescription()).uri(externalListingItem.getUri()) .abbreviation(externalListingItem.getAbbreviation()).status(eu.eudat.data.entities.Grant.Status.fromInteger(0)) - .source(externalListingItem.getSource()) + .source(externalListingItem.getTag()) .build(); grants.add(grant); } - grants.sort(Comparator.comparing(x -> x.getLabel())); + grants.sort(Comparator.comparing(Grant::getLabel)); return grants; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java index 9d16adfff..258e5ef2d 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java @@ -42,12 +42,12 @@ public class ProjectManager { .reference(externalListingItem.getRemoteId()).label(externalListingItem.getName()) .description(externalListingItem.getDescription()).uri(externalListingItem.getUri()) .abbreviation(externalListingItem.getAbbreviation()).status(eu.eudat.data.entities.Project.Status.fromInteger(0)) - .source(externalListingItem.getSource()) + .source(externalListingItem.getTag()) .build(); projects.add(project); } - projects.sort(Comparator.comparing(x -> x.getLabel())); + projects.sort(Comparator.comparing(Project::getLabel)); return projects; } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java index 15992ef60..9a9912c5a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java @@ -10,6 +10,7 @@ public class UrlConfiguration { private Integer ordinal; private String url; private DataUrlConfiguration data; + private String type; private String paginationPath; public String getKey() { @@ -66,4 +67,12 @@ public class UrlConfiguration { this.paginationPath = paginationPath; } + public String getType() { + return type; + } + + @XmlElement(name = "type") + public void setType(String type) { + this.type = type; + } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index 0ebd415e0..c10ed37bc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -1,10 +1,9 @@ package eu.eudat.logic.proxy.fetching; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; -import eu.eudat.configurations.dynamicfunder.DynamicFunderConfiguration; -import eu.eudat.configurations.dynamicgrant.DynamicGrantConfiguration; -import eu.eudat.configurations.dynamicproject.DynamicProjectConfiguration; import eu.eudat.logic.proxy.config.DataUrlConfiguration; import eu.eudat.logic.proxy.config.FetchStrategy; import eu.eudat.logic.proxy.config.UrlConfiguration; @@ -12,9 +11,11 @@ import eu.eudat.logic.proxy.config.configloaders.ConfigLoader; import eu.eudat.logic.proxy.config.exceptions.HugeResultSet; import eu.eudat.logic.proxy.config.exceptions.NoURLFound; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.stereotype.Service; +import java.io.File; import java.io.IOException; import java.net.HttpURLConnection; import java.net.MalformedURLException; @@ -27,16 +28,12 @@ public class RemoteFetcher { private ConfigLoader configLoader; - private DynamicGrantConfiguration dynamicGrantConfiguration; - private DynamicProjectConfiguration dynamicProjectConfiguration; - private DynamicFunderConfiguration dynamicFunderConfiguration; + @Value("${configuration.resources.path}") + private String resourcesPath; @Autowired - public RemoteFetcher(ConfigLoader configLoader, DynamicGrantConfiguration dynamicGrantConfiguration, DynamicProjectConfiguration dynamicProjectConfiguration, DynamicFunderConfiguration dynamicFunderConfiguration) { + public RemoteFetcher(ConfigLoader configLoader) { this.configLoader = configLoader; - this.dynamicGrantConfiguration = dynamicGrantConfiguration; - this.dynamicProjectConfiguration = dynamicProjectConfiguration; - this.dynamicFunderConfiguration = dynamicFunderConfiguration; } @Cacheable("repositories") @@ -50,21 +47,21 @@ public class RemoteFetcher { @Cacheable("grants") public List> getGrants(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = Arrays.asList(this.dynamicGrantConfiguration.getConfiguration().getMainExternalField().getUrlConfig()); + List urlConfigs = configLoader.getExternalUrls().getGrants().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("projects") public List> getProjects(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = Arrays.asList(this.dynamicProjectConfiguration.getConfiguration().getMainExternalField().getUrlConfig()); + List urlConfigs = configLoader.getExternalUrls().getProjects().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @Cacheable("funders") public List> getFunders(String query) throws NoURLFound, HugeResultSet { - List urlConfigs = Arrays.asList(this.dynamicFunderConfiguration.getConfiguration().getMainExternalField().getUrlConfig()); + List urlConfigs = configLoader.getExternalUrls().getFunders().getUrls(); FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode(); return getAll(urlConfigs, fetchStrategy, query); } @@ -132,7 +129,12 @@ public class RemoteFetcher { Collections.sort(urlConfigs, Comparator.comparing(UrlConfiguration::getOrdinal)); List> results = new LinkedList<>(); for (UrlConfiguration urlConfig : urlConfigs) { - results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel())); + if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { + results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel())); + } + else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { + results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query)); + } } return results; } @@ -221,7 +223,31 @@ public class RemoteFetcher { } return null; + } + private List> getAllResultsFromMockUpJson(String path, String query) { + String filePath = this.resourcesPath + path; + List> internalResults; + try { + ObjectMapper mapper = new ObjectMapper(); + internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); + searchListMap(internalResults, query); + return internalResults; + } catch (Exception e) { + e.printStackTrace(); + return new LinkedList<>(); + } + } + + private List> searchListMap(List> internalResults, String query) { + List> list = new LinkedList<>(); + for (Map map : internalResults) + { + if (map.get("name").toUpperCase().contains(query.toUpperCase())) { + list.add(map); + } + } + return list; } private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) { @@ -234,22 +260,21 @@ public class RemoteFetcher { } - class Results { - + static class Results { List> results; Map pagination; - public Results() { + Results() { this.results = new ArrayList>(); this.pagination = new HashMap(); } - public Results(List> results, Map pagination) { + Results(List> results, Map pagination) { this.results = results; this.pagination = pagination; } - public List> getResults() { + List> getResults() { return results; } @@ -257,15 +282,12 @@ public class RemoteFetcher { this.results = results; } - public Map getPagination() { + Map getPagination() { return pagination; } public void setPagination(Map pagination) { this.pagination = pagination; } - - } - } diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java index 3ceb97016..794d5c2dd 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java @@ -13,7 +13,7 @@ public class FundersExternalSourcesModel extends ExternalListingItemcristin 1 + External https://eestore.paas2.uninett.no/api/metadataschemarepo/ $['data'][*]['attributes'] @@ -26,6 +27,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/metadataschemarepo/ $['data'][*]['attributes'] @@ -50,6 +52,7 @@ cristin 1 + External https://eestore.paas2.uninett.no/api/tags/ $['data'][*]['attributes'] @@ -66,6 +69,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/tags/ $['data'][*]['attributes'] @@ -107,6 +111,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/projectrepo/ $['data'][*]['attributes'] @@ -133,6 +138,7 @@ cristin 1 + External https://eestore.paas2.uninett.no/api/projectrepo/ $['data'][*]['attributes'] @@ -149,6 +155,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/projectrepo/ $['data'][*]['attributes'] @@ -175,6 +182,7 @@ cristin 1 + External https://eestore.paas2.uninett.no/api/projectrepo/ $['data'][*]['attributes'] @@ -191,6 +199,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/projectrepo/ $['data'][*]['attributes'] @@ -217,6 +226,7 @@ cristin 1 + External https://eestore.paas2.uninett.no/api/datarepo/ $['data'][*]['attributes'] @@ -233,6 +243,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/datarepo/ $['data'][*]['attributes'] @@ -258,6 +269,7 @@ cristin 1 + External https://eestore.paas2.uninett.no/api/servicerepo/ $['data'][*]['attributes'] @@ -274,6 +286,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/servicerepo/ $['data'][*]['attributes'] @@ -298,6 +311,7 @@ cristin 1 + External https://eestore.paas2.uninett.no/api/personrepo/ $['data'][*]['attributes'] @@ -314,6 +328,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/personrepo/ $['data'][*]['attributes'] @@ -338,6 +353,7 @@ cristin 1 + External https://eestore.paas2.uninett.no/api/organizationrepo/ $['data'][*]['attributes'] @@ -354,6 +370,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/organizationrepo/ $['data'][*]['attributes'] @@ -378,6 +395,7 @@ cristin 1 + External https://eestore.paas2.uninett.no/api/datasetrepo/ $['data'][*]['attributes'] @@ -394,6 +412,7 @@ openAire 1 + External https://eestore.paas2.uninett.no/api/datasetrepo/ $['data'][*]['attributes'] diff --git a/dmp-backend/web/src/main/resources/FunderConfiguration.xml b/dmp-backend/web/src/main/resources/FunderConfiguration.xml deleted file mode 100644 index 3657195f7..000000000 --- a/dmp-backend/web/src/main/resources/FunderConfiguration.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - - field1 - DMP-EDITOR.FIELDS.PROJECT - http://localhost:9091/api/project/ - search - id - name - - - field2 - funder - - - false - - - field2 - DMP-EDITOR.FIELDS.FUNDER - http://localhost:9091/api/funder/ - search - id - name - - - false - - - field3 - DMP-EDITOR.FIELDS.GRANT - http://localhost:9091/api/grant/ - search - id - name - - - false - - - - field3 - project.configuration.grant.name - - 1 - https://eestore.paas2.uninett.no/api/projectrepo/ - - $['data'][*]['attributes'] - - 'pid' - 'name' - 'uri' - 'description' - 'source' - - - $['meta']['pagination']['page','pages','count'] - - pid - name - - - navbar - NAV-BAR.PROJECTS - - - listingTitle - PROJECT-LISTING.TITLE - - - editorTitle - PROJECT-EDITOR.TITLE.NEW - - - editorLogo - PROJECT-EDITOR.FIELDS.LOGO - - - dmpEditor - DMP-EDITOR.FIELDS.PROJECT - - - criteriaStart - CRITERIA.PROJECTS.PERIOD-FROM - - - criteriaEnd - CRITERIA.PROJECTS.PERIOD-TO - - - dmpCriteria - CRITERIA.DMP.PROJECTS - - - - - - - \ No newline at end of file diff --git a/dmp-backend/web/src/main/resources/ProjectConfiguration.xml b/dmp-backend/web/src/main/resources/ProjectConfiguration.xml deleted file mode 100644 index 3657195f7..000000000 --- a/dmp-backend/web/src/main/resources/ProjectConfiguration.xml +++ /dev/null @@ -1,99 +0,0 @@ - - - - field1 - DMP-EDITOR.FIELDS.PROJECT - http://localhost:9091/api/project/ - search - id - name - - - field2 - funder - - - false - - - field2 - DMP-EDITOR.FIELDS.FUNDER - http://localhost:9091/api/funder/ - search - id - name - - - false - - - field3 - DMP-EDITOR.FIELDS.GRANT - http://localhost:9091/api/grant/ - search - id - name - - - false - - - - field3 - project.configuration.grant.name - - 1 - https://eestore.paas2.uninett.no/api/projectrepo/ - - $['data'][*]['attributes'] - - 'pid' - 'name' - 'uri' - 'description' - 'source' - - - $['meta']['pagination']['page','pages','count'] - - pid - name - - - navbar - NAV-BAR.PROJECTS - - - listingTitle - PROJECT-LISTING.TITLE - - - editorTitle - PROJECT-EDITOR.TITLE.NEW - - - editorLogo - PROJECT-EDITOR.FIELDS.LOGO - - - dmpEditor - DMP-EDITOR.FIELDS.PROJECT - - - criteriaStart - CRITERIA.PROJECTS.PERIOD-FROM - - - criteriaEnd - CRITERIA.PROJECTS.PERIOD-TO - - - dmpCriteria - CRITERIA.DMP.PROJECTS - - - - - - - \ No newline at end of file diff --git a/dmp-backend/web/src/main/resources/application-staging.properties b/dmp-backend/web/src/main/resources/application-staging.properties index b797815d4..08b7a4002 100644 --- a/dmp-backend/web/src/main/resources/application-staging.properties +++ b/dmp-backend/web/src/main/resources/application-staging.properties @@ -21,10 +21,10 @@ configuration.dynamicGrantUrl=/tmp/GrantConfiguration.xml configuration.dynamicProjectUrl=/tmp/ProjectConfiguration.xml configuration.dynamicFunderUrl=/tmp/FunderConfiguration.xml configuration.h2020template=/tmp/h2020.docx -configuration.resources.path=/tmp +configuration.resources.path=/tmp/ ####################MOCK FILES CONFIGURATIONS########## -dataset.tags.mock=/mockupTags.json +dataset.tags.mock=mockupTags.json ####################SPRING MAIL CONFIGURATIONS################# spring.mail.default-encoding=UTF-8