Implements the functionality of being able to add static data when using external services, and refactors Funder, Grant and Project to be configured by "ExternalUrs". (Issue #147)

This commit is contained in:
gkolokythas 2019-09-05 17:44:25 +03:00
parent f5d2fca682
commit 9a90e3f8af
15 changed files with 93 additions and 241 deletions

View File

@ -11,7 +11,7 @@ public class MainProperty {
private String name;
private String queryProperty;
private String externalFieldId;
private UrlConfiguration urlConfig;
private List<UrlConfiguration> urlConfig;
private String externalFieldLabel;
private List<Dependency> dependencies;
private Boolean required;
@ -82,12 +82,12 @@ public class MainProperty {
this.queryProperty = queryProperty;
}
public UrlConfiguration getUrlConfig() {
public List<UrlConfiguration> getUrlConfig() {
return urlConfig;
}
@XmlElement(name = "urlConfig")
public void setUrlConfig(UrlConfiguration urlConfig) {
public void setUrlConfig(List<UrlConfiguration> urlConfig) {
this.urlConfig = urlConfig;
}

View File

@ -14,7 +14,7 @@ public class MainProperty {
private String name;
private String queryProperty;
private String externalFieldId;
private UrlConfiguration urlConfig;
private List<UrlConfiguration> urlConfig;
private String externalFieldLabel;
private List<Dependency> dependencies;
private Boolean required;
@ -85,12 +85,12 @@ public class MainProperty {
this.queryProperty = queryProperty;
}
public UrlConfiguration getUrlConfig() {
public List<UrlConfiguration> getUrlConfig() {
return urlConfig;
}
@XmlElement(name = "urlConfig")
public void setUrlConfig(UrlConfiguration urlConfig) {
public void setUrlConfig(List<UrlConfiguration> urlConfig) {
this.urlConfig = urlConfig;
}

View File

@ -11,7 +11,7 @@ public class MainProperty {
private String name;
private String queryProperty;
private String externalFieldId;
private UrlConfiguration urlConfig;
private List<UrlConfiguration> urlConfig;
private String externalFieldLabel;
private List<Dependency> dependencies;
private Boolean required;
@ -82,12 +82,12 @@ public class MainProperty {
this.queryProperty = queryProperty;
}
public UrlConfiguration getUrlConfig() {
public List<UrlConfiguration> getUrlConfig() {
return urlConfig;
}
@XmlElement(name = "urlConfig")
public void setUrlConfig(UrlConfiguration urlConfig) {
public void setUrlConfig(List<UrlConfiguration> urlConfig) {
this.urlConfig = urlConfig;
}

View File

@ -41,12 +41,12 @@ public class FunderManager {
eu.eudat.models.data.funder.Funder funder = apiContext.getOperationsContext().getBuilderFactory().getBuilder(FunderBuilder.class)
.reference(externalListingItem.getRemoteId()).label(externalListingItem.getName())
.status(eu.eudat.data.entities.Funder.Status.fromInteger(0))
.source(externalListingItem.getSource())
.source(externalListingItem.getTag())
.build();
funders.add(funder);
}
funders.sort(Comparator.comparing(x -> x.getLabel()));
funders.sort(Comparator.comparing(Funder::getLabel));
return funders;
}
}

View File

@ -129,12 +129,12 @@ public class GrantManager {
.reference(externalListingItem.getRemoteId()).label(externalListingItem.getName())
.description(externalListingItem.getDescription()).uri(externalListingItem.getUri())
.abbreviation(externalListingItem.getAbbreviation()).status(eu.eudat.data.entities.Grant.Status.fromInteger(0))
.source(externalListingItem.getSource())
.source(externalListingItem.getTag())
.build();
grants.add(grant);
}
grants.sort(Comparator.comparing(x -> x.getLabel()));
grants.sort(Comparator.comparing(Grant::getLabel));
return grants;
}

View File

@ -42,12 +42,12 @@ public class ProjectManager {
.reference(externalListingItem.getRemoteId()).label(externalListingItem.getName())
.description(externalListingItem.getDescription()).uri(externalListingItem.getUri())
.abbreviation(externalListingItem.getAbbreviation()).status(eu.eudat.data.entities.Project.Status.fromInteger(0))
.source(externalListingItem.getSource())
.source(externalListingItem.getTag())
.build();
projects.add(project);
}
projects.sort(Comparator.comparing(x -> x.getLabel()));
projects.sort(Comparator.comparing(Project::getLabel));
return projects;
}
}

View File

@ -10,6 +10,7 @@ public class UrlConfiguration {
private Integer ordinal;
private String url;
private DataUrlConfiguration data;
private String type;
private String paginationPath;
public String getKey() {
@ -66,4 +67,12 @@ public class UrlConfiguration {
this.paginationPath = paginationPath;
}
public String getType() {
return type;
}
@XmlElement(name = "type")
public void setType(String type) {
this.type = type;
}
}

View File

@ -1,10 +1,9 @@
package eu.eudat.logic.proxy.fetching;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.configurations.dynamicfunder.DynamicFunderConfiguration;
import eu.eudat.configurations.dynamicgrant.DynamicGrantConfiguration;
import eu.eudat.configurations.dynamicproject.DynamicProjectConfiguration;
import eu.eudat.logic.proxy.config.DataUrlConfiguration;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
@ -12,9 +11,11 @@ import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
@ -27,16 +28,12 @@ public class RemoteFetcher {
private ConfigLoader configLoader;
private DynamicGrantConfiguration dynamicGrantConfiguration;
private DynamicProjectConfiguration dynamicProjectConfiguration;
private DynamicFunderConfiguration dynamicFunderConfiguration;
@Value("${configuration.resources.path}")
private String resourcesPath;
@Autowired
public RemoteFetcher(ConfigLoader configLoader, DynamicGrantConfiguration dynamicGrantConfiguration, DynamicProjectConfiguration dynamicProjectConfiguration, DynamicFunderConfiguration dynamicFunderConfiguration) {
public RemoteFetcher(ConfigLoader configLoader) {
this.configLoader = configLoader;
this.dynamicGrantConfiguration = dynamicGrantConfiguration;
this.dynamicProjectConfiguration = dynamicProjectConfiguration;
this.dynamicFunderConfiguration = dynamicFunderConfiguration;
}
@Cacheable("repositories")
@ -50,21 +47,21 @@ public class RemoteFetcher {
@Cacheable("grants")
public List<Map<String, String>> getGrants(String query) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = Arrays.asList(this.dynamicGrantConfiguration.getConfiguration().getMainExternalField().getUrlConfig());
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getGrants().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("projects")
public List<Map<String, String>> getProjects(String query) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = Arrays.asList(this.dynamicProjectConfiguration.getConfiguration().getMainExternalField().getUrlConfig());
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@Cacheable("funders")
public List<Map<String, String>> getFunders(String query) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = Arrays.asList(this.dynamicFunderConfiguration.getConfiguration().getMainExternalField().getUrlConfig());
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
return getAll(urlConfigs, fetchStrategy, query);
}
@ -132,7 +129,12 @@ public class RemoteFetcher {
Collections.sort(urlConfigs, Comparator.comparing(UrlConfiguration::getOrdinal));
List<Map<String, String>> results = new LinkedList<>();
for (UrlConfiguration urlConfig : urlConfigs) {
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel()));
if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), query, urlConfig.getLabel()));
}
else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), query));
}
}
return results;
}
@ -221,7 +223,31 @@ public class RemoteFetcher {
}
return null;
}
private List<Map<String, String>> getAllResultsFromMockUpJson(String path, String query) {
String filePath = this.resourcesPath + path;
List<Map<String, String>> internalResults;
try {
ObjectMapper mapper = new ObjectMapper();
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, Object>>>(){});
searchListMap(internalResults, query);
return internalResults;
} catch (Exception e) {
e.printStackTrace();
return new LinkedList<>();
}
}
private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
List<Map<String, String>> list = new LinkedList<>();
for (Map<String, String> map : internalResults)
{
if (map.get("name").toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
}
return list;
}
private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
@ -234,22 +260,21 @@ public class RemoteFetcher {
}
class Results {
static class Results {
List<Map<String, String>> results;
Map<String, Integer> pagination;
public Results() {
Results() {
this.results = new ArrayList<Map<String, String>>();
this.pagination = new HashMap<String, Integer>();
}
public Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
this.results = results;
this.pagination = pagination;
}
public List<Map<String, String>> getResults() {
List<Map<String, String>> getResults() {
return results;
}
@ -257,15 +282,12 @@ public class RemoteFetcher {
this.results = results;
}
public Map<String, Integer> getPagination() {
Map<String, Integer> getPagination() {
return pagination;
}
public void setPagination(Map<String, Integer> pagination) {
this.pagination = pagination;
}
}
}

View File

@ -13,7 +13,7 @@ public class FundersExternalSourcesModel extends ExternalListingItem<FundersExte
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setDescription(item.get("description"));
model.setSource(item.get("source"));
model.setTag(item.get("tag"));
this.add(model);
}
return this;

View File

@ -13,7 +13,7 @@ public class GrantsExternalSourcesModel extends ExternalListingItem<GrantsExtern
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setDescription(item.get("description"));
model.setSource(item.get("source"));
model.setTag(item.get("tag"));
this.add(model);
}
return this;

View File

@ -12,7 +12,7 @@ public class ProjectsExternalSourcesModel extends ExternalListingItem<ProjectsEx
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setDescription(item.get("description"));
model.setSource(item.get("source"));
model.setTag(item.get("tag"));
this.add(model);
}
return this;

View File

@ -10,6 +10,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/metadataschemarepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -26,6 +27,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/metadataschemarepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -50,6 +52,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/tags/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -66,6 +69,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/tags/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -107,6 +111,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -133,6 +138,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -149,6 +155,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -175,6 +182,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -191,6 +199,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -217,6 +226,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/datarepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -233,6 +243,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/datarepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -258,6 +269,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/servicerepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -274,6 +286,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/servicerepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -298,6 +311,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/personrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -314,6 +328,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/personrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -338,6 +353,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/organizationrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -354,6 +370,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/organizationrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -378,6 +395,7 @@
<key>cristin</key>
<label>Cristin</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/datasetrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
@ -394,6 +412,7 @@
<key>openAire</key>
<label>OpenAIRE</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://eestore.paas2.uninett.no/api/datasetrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>

View File

@ -1,99 +0,0 @@
<configuration>
<configurationProperties>
<property>
<id>field1</id>
<name>DMP-EDITOR.FIELDS.PROJECT</name>
<sourceUrl>http://localhost:9091/api/project/</sourceUrl>
<queryProperty>search</queryProperty>
<externalFieldId>id</externalFieldId>
<externalFieldLabel>name</externalFieldLabel>
<dependencies>
<dependency>
<id>field2</id>
<queryProperty>funder</queryProperty>
</dependency>
</dependencies>
<required>false</required>
</property>
<property>
<id>field2</id>
<name>DMP-EDITOR.FIELDS.FUNDER</name>
<sourceUrl>http://localhost:9091/api/funder/</sourceUrl>
<queryProperty>search</queryProperty>
<externalFieldId>id</externalFieldId>
<externalFieldLabel>name</externalFieldLabel>
<dependencies>
</dependencies>
<required>false</required>
</property>
<property>
<id>field3</id>
<name>DMP-EDITOR.FIELDS.GRANT</name>
<sourceUrl>http://localhost:9091/api/grant/</sourceUrl>
<queryProperty>search</queryProperty>
<externalFieldId>id</externalFieldId>
<externalFieldLabel>name</externalFieldLabel>
<dependencies>
</dependencies>
<required>false</required>
</property>
</configurationProperties>
<mainExternalField>
<id>field3</id>
<name>project.configuration.grant.name</name>
<urlConfig>
<ordinal>1</ordinal>
<url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
<fields>
<id>'pid'</id>
<name>'name'</name>
<uri>'uri'</uri>
<description>'description'</description>
<source>'source'</source>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
<externalFieldId>pid</externalFieldId>
<externalFieldLabel>name</externalFieldLabel>
<language>
<languageProperty>
<key>navbar</key>
<languageKey>NAV-BAR.PROJECTS</languageKey>
</languageProperty>
<languageProperty>
<key>listingTitle</key>
<languageKey>PROJECT-LISTING.TITLE</languageKey>
</languageProperty>
<languageProperty>
<key>editorTitle</key>
<languageKey>PROJECT-EDITOR.TITLE.NEW</languageKey>
</languageProperty>
<languageProperty>
<key>editorLogo</key>
<languageKey>PROJECT-EDITOR.FIELDS.LOGO</languageKey>
</languageProperty>
<languageProperty>
<key>dmpEditor</key>
<languageKey>DMP-EDITOR.FIELDS.PROJECT</languageKey>
</languageProperty>
<languageProperty>
<key>criteriaStart</key>
<languageKey>CRITERIA.PROJECTS.PERIOD-FROM</languageKey>
</languageProperty>
<languageProperty>
<key>criteriaEnd</key>
<languageKey>CRITERIA.PROJECTS.PERIOD-TO</languageKey>
</languageProperty>
<languageProperty>
<key>dmpCriteria</key>
<languageKey>CRITERIA.DMP.PROJECTS</languageKey>
</languageProperty>
</language>
<dependencies>
</dependencies>
<required></required>
</mainExternalField>
</configuration>

View File

@ -1,99 +0,0 @@
<configuration>
<configurationProperties>
<property>
<id>field1</id>
<name>DMP-EDITOR.FIELDS.PROJECT</name>
<sourceUrl>http://localhost:9091/api/project/</sourceUrl>
<queryProperty>search</queryProperty>
<externalFieldId>id</externalFieldId>
<externalFieldLabel>name</externalFieldLabel>
<dependencies>
<dependency>
<id>field2</id>
<queryProperty>funder</queryProperty>
</dependency>
</dependencies>
<required>false</required>
</property>
<property>
<id>field2</id>
<name>DMP-EDITOR.FIELDS.FUNDER</name>
<sourceUrl>http://localhost:9091/api/funder/</sourceUrl>
<queryProperty>search</queryProperty>
<externalFieldId>id</externalFieldId>
<externalFieldLabel>name</externalFieldLabel>
<dependencies>
</dependencies>
<required>false</required>
</property>
<property>
<id>field3</id>
<name>DMP-EDITOR.FIELDS.GRANT</name>
<sourceUrl>http://localhost:9091/api/grant/</sourceUrl>
<queryProperty>search</queryProperty>
<externalFieldId>id</externalFieldId>
<externalFieldLabel>name</externalFieldLabel>
<dependencies>
</dependencies>
<required>false</required>
</property>
</configurationProperties>
<mainExternalField>
<id>field3</id>
<name>project.configuration.grant.name</name>
<urlConfig>
<ordinal>1</ordinal>
<url>https://eestore.paas2.uninett.no/api/projectrepo/</url>
<data>
<path>$['data'][*]['attributes']</path>
<fields>
<id>'pid'</id>
<name>'name'</name>
<uri>'uri'</uri>
<description>'description'</description>
<source>'source'</source>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
<externalFieldId>pid</externalFieldId>
<externalFieldLabel>name</externalFieldLabel>
<language>
<languageProperty>
<key>navbar</key>
<languageKey>NAV-BAR.PROJECTS</languageKey>
</languageProperty>
<languageProperty>
<key>listingTitle</key>
<languageKey>PROJECT-LISTING.TITLE</languageKey>
</languageProperty>
<languageProperty>
<key>editorTitle</key>
<languageKey>PROJECT-EDITOR.TITLE.NEW</languageKey>
</languageProperty>
<languageProperty>
<key>editorLogo</key>
<languageKey>PROJECT-EDITOR.FIELDS.LOGO</languageKey>
</languageProperty>
<languageProperty>
<key>dmpEditor</key>
<languageKey>DMP-EDITOR.FIELDS.PROJECT</languageKey>
</languageProperty>
<languageProperty>
<key>criteriaStart</key>
<languageKey>CRITERIA.PROJECTS.PERIOD-FROM</languageKey>
</languageProperty>
<languageProperty>
<key>criteriaEnd</key>
<languageKey>CRITERIA.PROJECTS.PERIOD-TO</languageKey>
</languageProperty>
<languageProperty>
<key>dmpCriteria</key>
<languageKey>CRITERIA.DMP.PROJECTS</languageKey>
</languageProperty>
</language>
<dependencies>
</dependencies>
<required></required>
</mainExternalField>
</configuration>

View File

@ -21,10 +21,10 @@ configuration.dynamicGrantUrl=/tmp/GrantConfiguration.xml
configuration.dynamicProjectUrl=/tmp/ProjectConfiguration.xml
configuration.dynamicFunderUrl=/tmp/FunderConfiguration.xml
configuration.h2020template=/tmp/h2020.docx
configuration.resources.path=/tmp
configuration.resources.path=/tmp/
####################MOCK FILES CONFIGURATIONS##########
dataset.tags.mock=/mockupTags.json
dataset.tags.mock=mockupTags.json
####################SPRING MAIL CONFIGURATIONS#################
spring.mail.default-encoding=UTF-8