MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Add OpenAIRE source for external Datasets

This commit is contained in:
George Kalampokis 2020-07-06 11:40:00 +03:00
parent 0fefc762a6
commit 7f56afad5f
23 changed files with 148 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataRepositoryManager.java
View File

@ -38,7 +38,7 @@ public class DataRepositoryManager {
public List<DataRepositoryModel> getDataRepositories(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRepositories(externalUrlCriteria, type);
List<Map<String, Object>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRepositories(externalUrlCriteria, type);
DataRepositoryCriteria criteria = new DataRepositoryCriteria();
if (!query.isEmpty()) criteria.setLike(query);

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ExternalDatasetManager.java
View File

@ -57,7 +57,7 @@ public class ExternalDatasetManager {
// Fetch external Datasets from external sources.
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = remoteFetcher.getDatasets(externalUrlCriteria, type);
List<Map<String, Object>> remoteRepos = remoteFetcher.getDatasets(externalUrlCriteria, type);
// Parse items from external sources to listing models.
ObjectMapper mapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java
View File

@ -42,7 +42,7 @@ public class FunderManager {
QueryableList<eu.eudat.data.entities.Funder> authItems = apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getAuthenticated(items, userInfo);
List<Funder> funders = authItems.select(item -> new eu.eudat.models.data.funder.Funder().fromDataModel(item));
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(funderCriteria.getCriteria().getLike());
List<Map<String, String>> remoteRepos = remoteFetcher.getFunders(externalUrlCriteria);
List<Map<String, Object>> remoteRepos = remoteFetcher.getFunders(externalUrlCriteria);
FundersExternalSourcesModel fundersExternalSourcesModel = new FundersExternalSourcesModel().fromExternalItem(remoteRepos);
for (ExternalSourcesItemModel externalListingItem : fundersExternalSourcesModel) {
eu.eudat.models.data.funder.Funder funder = apiContext.getOperationsContext().getBuilderFactory().getBuilder(FunderBuilder.class)

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java
View File

@ -126,7 +126,7 @@ public class GrantManager {
QueryableList<eu.eudat.data.entities.Grant> authItems = apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getAuthenticated(items, userInfo);
List<eu.eudat.models.data.grant.Grant> grants = authItems.select(item -> new Grant().fromDataModel(item));
List<Map<String, String>> remoteRepos = remoteFetcher.getGrants(externalUrlCriteria);
List<Map<String, Object>> remoteRepos = remoteFetcher.getGrants(externalUrlCriteria);
GrantsExternalSourcesModel grantsExternalSourcesModel = new GrantsExternalSourcesModel().fromExternalItem(remoteRepos);
for (ExternalSourcesItemModel externalListingItem : grantsExternalSourcesModel) {

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/LicenseManager.java
View File

@ -33,7 +33,7 @@ public class LicenseManager {
public List<LicenseModel> getLicenses(String query, String type) throws HugeResultSet, NoURLFound {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getlicenses(externalUrlCriteria, type);
List<Map<String, Object>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getlicenses(externalUrlCriteria, type);
DataRepositoryCriteria criteria = new DataRepositoryCriteria();
if (!query.isEmpty()) criteria.setLike(query);

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/OrganisationsManager.java
View File

@ -68,7 +68,7 @@ public class OrganisationsManager {
public List<Organisation> getCriteriaWithExternal(String query, String type) throws HugeResultSet, NoURLFound {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = apiContext.getOperationsContext().getRemoteFetcher().getOrganisations(externalUrlCriteria, type);
List<Map<String, Object>> remoteRepos = apiContext.getOperationsContext().getRemoteFetcher().getOrganisations(externalUrlCriteria, type);
OrganisationsExternalSourcesModel organisationsExternalSourcesModel = new OrganisationsExternalSourcesModel().fromExternalItem(remoteRepos);
List<Organisation> organisations = new LinkedList<>();
for (ExternalSourcesItemModel externalListingItem : organisationsExternalSourcesModel) {

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java
View File

@ -41,7 +41,7 @@ public class ProjectManager {
QueryableList<eu.eudat.data.entities.Project> authItems = apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getAuthenticated(items, userInfo);
List<Project> projects = authItems.select(item -> new Project().fromDataModel(item));
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(projectCriteria.getCriteria().getLike());
List<Map<String, String>> remoteRepos = remoteFetcher.getProjects(externalUrlCriteria);
List<Map<String, Object>> remoteRepos = remoteFetcher.getProjects(externalUrlCriteria);
ProjectsExternalSourcesModel projectsExternalSourcesModel = new ProjectsExternalSourcesModel().fromExternalItem(remoteRepos);
for (ExternalSourcesItemModel externalListingItem : projectsExternalSourcesModel) {
eu.eudat.models.data.project.Project project = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ProjectBuilder.class)

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/RegistryManager.java
View File

@ -39,7 +39,7 @@ public class RegistryManager {
public List<RegistryModel> getRegistries(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRegistries(externalUrlCriteria, type);
List<Map<String, Object>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getRegistries(externalUrlCriteria, type);
RegistryCriteria criteria = new RegistryCriteria();
if (!query.isEmpty()) criteria.setLike(query);

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ResearcherManager.java
View File

@ -57,7 +57,7 @@ public class ResearcherManager {
item.setTag(keyToSourceMap.get(item.getKey()));
}
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(researcherCriteriaRequest.getCriteria().getName());
List<Map<String, String>> remoteRepos = remoteFetcher.getResearchers(externalUrlCriteria,null);
List<Map<String, Object>> remoteRepos = remoteFetcher.getResearchers(externalUrlCriteria,null);
ResearchersExternalSourcesModel researchersExternalSourcesModel = new ResearchersExternalSourcesModel().fromExternalItem(remoteRepos);
for (ExternalSourcesItemModel externalListingItem : researchersExternalSourcesModel) {
eu.eudat.models.data.dmp.Researcher researcher = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ResearcherBuilder.class)

2
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ServiceManager.java
View File

@ -36,7 +36,7 @@ public class ServiceManager {
public List<ServiceModel> getServices(String query, String type, Principal principal) throws HugeResultSet, NoURLFound {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getServices(externalUrlCriteria, type);
List<Map<String, Object>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().getServices(externalUrlCriteria, type);
ServiceCriteria criteria = new ServiceCriteria();
if (!query.isEmpty()) criteria.setLike(query);

88
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -43,7 +43,7 @@ public class RemoteFetcher {
}
@Cacheable("repositories")
public List<Map<String, String>> getRepositories(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getRepositories(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRepositories().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getRepositories().getUrls();
@ -52,28 +52,28 @@ public class RemoteFetcher {
}
@Cacheable("grants")
public List<Map<String, String>> getGrants(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getGrants(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getGrants().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getGrants().getFetchMode();
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("projects")
public List<Map<String, String>> getProjects(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getProjects(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getProjects().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getProjects().getFetchMode();
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("funders")
public List<Map<String, String>> getFunders(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getFunders(ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs = configLoader.getExternalUrls().getFunders().getUrls();
FetchStrategy fetchStrategy = configLoader.getExternalUrls().getFunders().getFetchMode();
return getAll(urlConfigs, fetchStrategy, externalUrlCriteria);
}
@Cacheable("organisations")
public List<Map<String, String>> getOrganisations(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getOrganisations(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getOrganisations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getOrganisations().getUrls();
@ -82,7 +82,7 @@ public class RemoteFetcher {
}
@Cacheable("registries")
public List<Map<String, String>> getRegistries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getRegistries(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getRegistries().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getRegistries().getUrls();
@ -91,7 +91,7 @@ public class RemoteFetcher {
}
@Cacheable("services")
public List<Map<String, String>> getServices(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getServices(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getServices().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getServices().getUrls();
@ -100,7 +100,7 @@ public class RemoteFetcher {
}
@Cacheable("researchers")
public List<Map<String, String>> getResearchers(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getResearchers(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getResearchers().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getResearchers().getUrls();
@ -118,7 +118,7 @@ public class RemoteFetcher {
}*/
@Cacheable("externalDatasets")
public List<Map<String, String>> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getDatasets(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getDatasets().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getDatasets().getUrls();
@ -127,7 +127,7 @@ public class RemoteFetcher {
}
@Cacheable("licenses")
public List<Map<String, String>> getlicenses(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
public List<Map<String, Object>> getlicenses(ExternalUrlCriteria externalUrlCriteria, String key) throws NoURLFound, HugeResultSet {
List<UrlConfiguration> urlConfigs =
key != null && !key.isEmpty() ? configLoader.getExternalUrls().getLicenses().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
: configLoader.getExternalUrls().getLicenses().getUrls();
@ -136,9 +136,9 @@ public class RemoteFetcher {
}
private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
private List<Map<String, Object>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) throws NoURLFound, HugeResultSet {
List<Map<String, String>> results = new LinkedList<>();
List<Map<String, Object>> results = new LinkedList<>();
if (urlConfigs == null || urlConfigs.isEmpty()) return results;
// throw new NoURLFound("No Repository urls found in configuration");
@ -217,7 +217,7 @@ public class RemoteFetcher {
return completedPath;
}
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage) throws HugeResultSet {
private List<Map<String, Object>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage) throws HugeResultSet {
Set<Integer> pages = new HashSet<>();
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage);
@ -277,12 +277,26 @@ public class RemoteFetcher {
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
new HashMap<>(1, 1));
List<Map<String, String>> fixedResults = results.getResults().stream().map(item -> {
List<Map<String, Object>> fixedResults = results.getResults().stream().map(item -> {
String id = jsonDataPath.getFieldsUrlConfiguration().getId().replace("'", "");
String name = jsonDataPath.getFieldsUrlConfiguration().getName().replace("'", "");
if (! (item.get(id) instanceof String)) {
Object obj = item.get(id);
JSONArray jarr = (JSONArray) obj;
item.put(id, jarr.get(0).toString());
if (item.get(id) instanceof LinkedHashMap) {
item.put(id, ((Map)item.get(id)).get("content"));
} else {
Object obj = item.get(id);
JSONArray jarr = (JSONArray) obj;
item.put(id, jarr.get(0).toString());
}
}
if (! (item.get(name) instanceof String)) {
if (item.get(name) instanceof LinkedHashMap) {
item.put(name, ((Map)item.get(name)).get("content"));
} else {
Object obj = item.get(name);
JSONArray jarr = (JSONArray) obj;
item.put(name, ((Map)jarr.get(1)).get("content"));
}
}
return item;
}).collect(Collectors.toList());
@ -294,10 +308,10 @@ public class RemoteFetcher {
+ "," + jsonDataPath.getFieldsUrlConfiguration().getHost() + "]"),
new HashMap<>(1, 1));
List<Map<String, String>> multiResults = results.results.stream().map(result -> {
List<Map<String, Object>> multiResults = results.results.stream().map(result -> {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria();
externalUrlCriteria.setPath(result.get("path"));
externalUrlCriteria.setHost(result.get("host"));
externalUrlCriteria.setPath((String) result.get("path"));
externalUrlCriteria.setHost((String) result.get("host"));
String replacedPath = replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalUrlCriteria, jsonDataPath.getUrlConfiguration().getFirstpage());
return getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType());
}).filter(Objects::nonNull).map(results1 -> results1.results.get(0)).collect(Collectors.toList());
@ -306,9 +320,9 @@ public class RemoteFetcher {
List<Map<String, Object>> tempRes = jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getId() + "," + jsonDataPath.getFieldsUrlConfiguration().getName()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getTypes() + "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "]");
List<Map<String, String>> finalRes = new ArrayList<>();
List<Map<String, Object>> finalRes = new ArrayList<>();
tempRes.forEach(map -> {
Map<String, String> resMap = new HashMap<>();
Map<String, Object> resMap = new HashMap<>();
map.forEach((key, value) -> {
if (key.equals(jsonDataPath.getFieldsUrlConfiguration().getTypes().substring(1, jsonDataPath.getFieldsUrlConfiguration().getTypes().length() - 1))) {
resMap.put("tags", ((JSONArray) value).toJSONString());
@ -343,7 +357,7 @@ public class RemoteFetcher {
reader = new PropertyDescriptor(jsonDataPath.getParseField(), aClass).getReadMethod();
}
ObjectMapper objectMapper = new ObjectMapper();
List<Map<String, String>> values = new ArrayList<>();
List<Map<String, Object>> values = new ArrayList<>();
int max = 1;
if (reader != null) {
Object invokedField = reader.invoke(data);
@ -363,10 +377,10 @@ public class RemoteFetcher {
} else {
value = data;
}
Map<String, String> map = objectMapper.convertValue(value, Map.class);
Map<String, Object> map = objectMapper.convertValue(value, Map.class);
if (jsonDataPath.getMergedFields() != null && !jsonDataPath.getMergedFields().isEmpty() && jsonDataPath.getMergedFieldName() != null && !jsonDataPath.getMergedFieldName().isEmpty()) {
Map<String, String> finalMap = new HashMap<>();
for (Map.Entry<String, String> entry : map.entrySet()) {
Map<String, Object> finalMap = new HashMap<>();
for (Map.Entry<String, Object> entry : map.entrySet()) {
if (jsonDataPath.getMergedFields().contains(entry.getKey())) {
if (!finalMap.containsKey(jsonDataPath.getMergedFieldName())) {
finalMap.put(jsonDataPath.getMergedFieldName(), entry.getValue());
@ -402,8 +416,8 @@ public class RemoteFetcher {
return null;
}
private List<Map<String, String>> getAllResultsFromMockUpJson(String path, String query) {
List<Map<String, String>> internalResults;
private List<Map<String, Object>> getAllResultsFromMockUpJson(String path, String query) {
List<Map<String, Object>> internalResults;
try {
String filePath = Paths.get(path).toUri().toURL().toString();
ObjectMapper mapper = new ObjectMapper();
@ -415,14 +429,14 @@ public class RemoteFetcher {
}
}
private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
List<Map<String, String>> list = new LinkedList<>();
for (Map<String, String> map : internalResults)
private List<Map<String, Object>> searchListMap(List<Map<String, Object>> internalResults, String query) {
List<Map<String, Object>> list = new LinkedList<>();
for (Map<String, Object> map : internalResults)
{
if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
if (map.get("name") != null && ((String)map.get("name")).toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
if (map.get("label") != null && ((String)map.get("label")).toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
}
@ -443,7 +457,7 @@ public class RemoteFetcher {
static class Results {
List<Map<String, String>> results;
List<Map<String, Object>> results;
Map<String, Integer> pagination;
Results() {
@ -451,16 +465,16 @@ public class RemoteFetcher {
this.pagination = new HashMap<>();
}
Results(List<Map<String, String>> results, Map<String, Integer> pagination) {
Results(List<Map<String, Object>> results, Map<String, Integer> pagination) {
this.results = results;
this.pagination = pagination;
}
List<Map<String, String>> getResults() {
List<Map<String, Object>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
public void setResults(List<Map<String, Object>> results) {
this.results = results;
}

12
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalDatasetSourcesModel.java vendored
View File

@ -6,13 +6,13 @@ import java.util.Map;
public class ExternalDatasetSourcesModel extends ExternalListingItem<ExternalDatasetSourcesModel> {
@Override
public ExternalDatasetSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public ExternalDatasetSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setId(item.get("pid"));
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setSource(item.get("source"));
model.setId((String)item.get("pid"));
model.setUri((String)item.get("uri"));
model.setName((String)item.get("name"));
model.setSource((String)item.get("source"));
this.add(model);
}
return this;

2
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ExternalItem.java vendored
View File

@ -5,5 +5,5 @@ import java.util.Map;
public interface ExternalItem<T> {
T fromExternalItem(List<Map<String, String>> values);
T fromExternalItem(List<Map<String, Object>> values);
}

18
dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java vendored
View File

@ -6,16 +6,16 @@ import java.util.Map;
public class FundersExternalSourcesModel extends ExternalListingItem<FundersExternalSourcesModel> {
@Override
public FundersExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public FundersExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setRemoteId(item.get("pid"));
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setDescription(item.get("description"));
model.setSource(item.get("source"));
model.setTag(item.get("tag"));
model.setKey(item.get("key"));
model.setRemoteId((String)item.get("pid"));
model.setUri((String)item.get("uri"));
model.setName((String)item.get("name"));
model.setDescription((String)item.get("description"));
model.setSource((String)item.get("source"));
model.setTag((String)item.get("tag"));
model.setKey((String)item.get("key"));
this.add(model);
}
return this;

16
dmp-backend/web/src/main/java/eu/eudat/models/data/external/GrantsExternalSourcesModel.java vendored
View File

@ -6,15 +6,15 @@ import java.util.Map;
public class GrantsExternalSourcesModel extends ExternalListingItem<GrantsExternalSourcesModel> {
@Override
public GrantsExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public GrantsExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setRemoteId(item.get("pid"));
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setDescription(item.get("description"));
model.setTag(item.get("tag"));
model.setKey(item.get("key"));
model.setRemoteId((String)item.get("pid"));
model.setUri((String)item.get("uri"));
model.setName((String)item.get("name"));
model.setDescription((String)item.get("description"));
model.setTag((String)item.get("tag"));
model.setKey((String)item.get("key"));
this.add(model);
}
return this;

10
dmp-backend/web/src/main/java/eu/eudat/models/data/external/LicensesExternalSourcesModel.java vendored
View File

@ -6,12 +6,12 @@ import java.util.Map;
public class LicensesExternalSourcesModel extends ExternalListingItem<LicensesExternalSourcesModel> {
@Override
public LicensesExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public LicensesExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setId(item.get("id"));
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setId((String)item.get("id"));
model.setUri((String)item.get("uri"));
model.setName((String)item.get("name"));
this.add(model);
}
return this;

14
dmp-backend/web/src/main/java/eu/eudat/models/data/external/OrganisationsExternalSourcesModel.java vendored
View File

@ -8,14 +8,14 @@ import java.util.Map;
@Component
public class OrganisationsExternalSourcesModel extends ExternalListingItem<OrganisationsExternalSourcesModel> {
@Override
public OrganisationsExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public OrganisationsExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setRemoteId(item.get("pid"));
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setTag(item.get("tag"));
model.setKey(item.get("key"));
model.setRemoteId((String)item.get("pid"));
model.setUri((String)item.get("uri"));
model.setName((String)item.get("name"));
model.setTag((String)item.get("tag"));
model.setKey((String)item.get("key"));
this.add(model);
}
return this;

16
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ProjectsExternalSourcesModel.java vendored
View File

@ -14,8 +14,8 @@ public class ProjectsExternalSourcesModel extends ExternalListingItem<ProjectsEx
private static final ObjectMapper mapper = new ObjectMapper();
@Override
public ProjectsExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public ProjectsExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
try {
JsonNode node = mapper.readTree(mapper.writeValueAsBytes(item));
@ -23,42 +23,42 @@ public class ProjectsExternalSourcesModel extends ExternalListingItem<ProjectsEx
if (name != null && !name.isNull() && name.isObject()) {
model.setName(node.get("name").get("$").asText());
} else {
model.setName(item.get("name"));
model.setName((String)item.get("name"));
}
JsonNode pid = node.get("pid");
if (pid != null && !pid.isNull() && pid.isObject()) {
model.setRemoteId(node.get("pid").get("$").asText());
} else {
model.setRemoteId(item.get("pid"));
model.setRemoteId((String)item.get("pid"));
}
JsonNode uri = node.get("uri");
if (uri != null && !uri.isNull() && uri.isObject()) {
model.setUri(node.get("uri").get("$").asText());
} else {
model.setUri(item.get("uri"));
model.setUri((String)item.get("uri"));
}
JsonNode description = node.get("description");
if (description != null && !description.isNull() && description.isObject()) {
model.setDescription(node.get("description").get("$").asText());
} else {
model.setDescription(item.get("description"));
model.setDescription((String)item.get("description"));
}
JsonNode tag = node.get("tag");
if (tag != null && !tag.isNull() && tag.isObject()) {
model.setTag(node.get("tag").get("$").asText());
} else {
model.setTag(item.get("tag"));
model.setTag((String)item.get("tag"));
}
JsonNode key = node.get("key");
if (key != null && !key.isNull() && key.isObject()) {
model.setKey(node.get("key").get("$").asText());
} else {
model.setKey(item.get("key"));
model.setKey((String)item.get("key"));
}
this.add(model);

12
dmp-backend/web/src/main/java/eu/eudat/models/data/external/RegistriesExternalSourcesModel.java vendored
View File

@ -6,13 +6,13 @@ import java.util.Map;
public class RegistriesExternalSourcesModel extends ExternalListingItem<RegistriesExternalSourcesModel> {
@Override
public RegistriesExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public RegistriesExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setId(item.get("pid"));
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setTag(item.get("tag"));
model.setId((String)item.get("pid"));
model.setUri((String)item.get("uri"));
model.setName((String)item.get("name"));
model.setTag((String)item.get("tag"));
this.add(model);
}
return this;

14
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ResearchersExternalSourcesModel.java vendored
View File

@ -6,14 +6,14 @@ import java.util.Map;
public class ResearchersExternalSourcesModel extends ExternalListingItem<ResearchersExternalSourcesModel> {
@Override
public ResearchersExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public ResearchersExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setRemoteId(item.get("pid"));
model.setUri(item.get("uri"));
model.setName(item.get("name"));
model.setTag(item.get("tag"));
model.setKey(item.get("key"));
model.setRemoteId((String)item.get("pid"));
model.setUri((String)item.get("uri"));
model.setName((String)item.get("name"));
model.setTag((String)item.get("tag"));
model.setKey((String)item.get("key"));
this.add(model);
}
return this;

12
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ServiceExternalSourcesModel.java vendored
View File

@ -6,13 +6,13 @@ import java.util.Map;
public class ServiceExternalSourcesModel extends ExternalListingItem<ServiceExternalSourcesModel> {
@Override
public ServiceExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public ServiceExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setId(item.get("pid"));
model.setUri(item.get("label"));
model.setName(item.get("name"));
model.setTag(item.get("tag"));
model.setId((String)item.get("pid"));
model.setUri((String)item.get("label"));
model.setName((String)item.get("name"));
model.setTag((String)item.get("tag"));
this.add(model);
}
return this;

12
dmp-backend/web/src/main/java/eu/eudat/models/data/external/TagExternalSourcesModel.java vendored
View File

@ -9,13 +9,13 @@ import java.util.Map;
public class TagExternalSourcesModel extends ExternalListingItem<TagExternalSourcesModel> {
@Override
public TagExternalSourcesModel fromExternalItem(List<Map<String, String>> values) {
for (Map<String, String> item : values) {
public TagExternalSourcesModel fromExternalItem(List<Map<String, Object>> values) {
for (Map<String, Object> item : values) {
ExternalSourcesItemModel model = new ExternalSourcesItemModel();
model.setId(item.get("pid"));
model.setUri(item.get("label"));
model.setName(item.get("name"));
model.setTag(item.get("tag"));
model.setId((String)item.get("pid"));
model.setUri((String)item.get("label"));
model.setName((String)item.get("name"));
model.setTag((String)item.get("tag"));
this.add(model);
}
return this;

18
dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml
View File

@ -897,6 +897,24 @@
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
<urlConfig>
<key>openaire</key>
<label>OpenAire</label>
<ordinal>2</ordinal>
<type>External</type>
<url>https://services.openaire.eu/search/v2/api/datasets/?q={like}&amp;page={page}&amp;size={pageSize}&amp;format=json</url>
<firstPage>0</firstPage>
<contenttype>application/json; charset=utf-8</contenttype>
<data>
<path>$['results'][*]['result']['metadata']['oaf:entity']['oaf:result']</path>
<fields>
<id>'pid'</id>
<name>'title'</name>
<count>'count'</count>
</fields>
</data>
<paginationpath>$['meta']['page','size','total']</paginationpath>
</urlConfig>
<!-- <urlConfig>
<key>openAire</key>
<label>OpenAIRE</label>