MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

reference fetcher changes

This commit is contained in:
Efstratios Giannopoulos 2024-02-12 17:43:41 +02:00
parent 87dd56ff2c
commit 957c1fad58
54 changed files with 923 additions and 2591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/AuthenticationConfigurationEntity.java
View File

@ -1,9 +1,10 @@
package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.service.remotefetcher.config.entities.AuthenticationConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
public class AuthenticationConfigurationEntity {
public class AuthenticationConfigurationEntity implements AuthenticationConfiguration {
private Boolean enabled;
private String authUrl;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/QueryConfigEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.QueryConfig;
import jakarta.xml.bind.annotation.XmlElement;
public class QueryConfigEntity {
public class QueryConfigEntity implements QueryConfig {
private String condition;
private String separator;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceBaseConfigurationEntity.java
View File

@ -1,12 +1,13 @@
package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public abstract class ReferenceTypeSourceBaseConfigurationEntity {
public abstract class ReferenceTypeSourceBaseConfigurationEntity implements SourceBaseConfiguration {
private String key;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceExternalApiConfigurationEntity.java
View File

@ -2,11 +2,12 @@ package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.service.remotefetcher.config.entities.SourceExternalApiConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity {
public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceExternalApiConfiguration<ResultsConfigurationEntity, AuthenticationConfigurationEntity, QueryConfigEntity> {
private String url;
private ResultsConfigurationEntity results;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceStaticOptionConfigurationEntity.java
View File

@ -1,11 +1,12 @@
package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.SourceStaticOptionConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity {
public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceStaticOptionConfiguration<ReferenceTypeStaticOptionEntity> {
List<ReferenceTypeStaticOptionEntity> options;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeStaticOptionEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.StaticOption;
import jakarta.xml.bind.annotation.XmlElement;
public class ReferenceTypeStaticOptionEntity {
public class ReferenceTypeStaticOptionEntity implements StaticOption {
private String code;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultFieldsMappingConfigurationEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.ResultFieldsMappingConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
public class ResultFieldsMappingConfigurationEntity {
public class ResultFieldsMappingConfigurationEntity implements ResultFieldsMappingConfiguration {
private String code;
private String responsePath;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultsConfigurationEntity.java
View File

@ -1,11 +1,12 @@
package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.ResultsConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ResultsConfigurationEntity {
public class ResultsConfigurationEntity implements ResultsConfiguration<ResultFieldsMappingConfigurationEntity> {
private String resultsArrayPath;
private List<ResultFieldsMappingConfigurationEntity> fieldsMapping;

58
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/DefinitionSearchBuilder.java
View File

@ -1,58 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Definition;
//import gr.cite.tools.data.builder.BuilderFactory;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class DefinitionSearchBuilder extends BaseBuilder<Definition, Map<String, String>> {
//
// private final BuilderFactory builderFactory;
// private final ReferenceTypeProperties properties;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public DefinitionSearchBuilder(
// ConventionService conventionService, BuilderFactory builderFactory, ReferenceTypeProperties properties) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(DefinitionSearchBuilder.class)));
// this.builderFactory = builderFactory;
// this.properties = properties;
// }
//
// public DefinitionSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Definition> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Definition> models = new ArrayList<>();
// //for (Map<String, String> d : data) {
// Definition m = new Definition();
// m.setFields(this.builderFactory.builder(FieldSearchBuilder.class).authorize(this.authorize).build(null, data));
// models.add(m);
// //}
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//}

88
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/FieldSearchBuilder.java
View File

@ -1,88 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.commons.enums.ReferenceFieldDataType;
//import eu.eudat.configurations.referencetype.ReferenceTypeField;
//import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Field;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class FieldSearchBuilder extends BaseBuilder<Field, Map<String, String>> {
//
// private final ReferenceTypeProperties properties;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public FieldSearchBuilder(
// ConventionService conventionService, ReferenceTypeProperties properties) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(FieldSearchBuilder.class)));
// this.properties = properties;
// }
//
// public FieldSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Field> build(FieldSet fields, List< Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Field> models = new ArrayList<>();
// for (Map<String, String> d : data) {
//
// ReferenceType referenceType = ReferenceType.valueOf(d.getOrDefault("referenceType", null));
// List<ReferenceTypeField> typeFields = this.getPropertiesFields(referenceType);
// if (typeFields.isEmpty()){
// return new ArrayList<>();
// }
//
// for (ReferenceTypeField typeField: typeFields){
// Field m = new Field();
// m.setCode(typeField.getCode());
// m.setDataType(ReferenceFieldDataType.valueOf(typeField.getDataType()));
// m.setValue(d.getOrDefault(typeField.getCode(), null));
// models.add(m);
// }
// }
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//
// private List<ReferenceTypeField> getPropertiesFields(ReferenceType referenceType){
// switch (referenceType) {
// case Taxonomies: return properties.getTaxonomy().get("fields");
// case Licenses: return properties.getLicence().get("fields");
// case Publications: return properties.getPublication().get("fields");
// case Journals: return properties.getJournal().get("fields");
// case PubRepositories: return properties.getPubRepository().get("fields");
// case DataRepositories: return properties.getDataRepository().get("fields");
// case Registries: return properties.getRegistry().get("fields");
// case Services: return properties.getService().get("fields");
// case Organizations: return properties.getOrganisation().get("fields");
// case Datasets: return properties.getDataset().get("fields");
// case Funder: return properties.getFunder().get("fields");
// case Grants: return properties.getGrant().get("fields");
// case Project: return properties.getProject().get("fields");
// case Researcher: return properties.getResearcher().get("fields");
// default: return null;
// }
// }
//}

96
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/ReferenceSearchBuilder.java
View File

@ -1,96 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.commons.enums.ReferenceSourceType;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.Reference;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Definition;
//import eu.eudat.model.referencedefinition.Field;
//import gr.cite.tools.data.builder.BuilderFactory;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class ReferenceSearchBuilder extends BaseBuilder<Reference, Map<String, String>> {
//
// private final BuilderFactory builderFactory;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public ReferenceSearchBuilder(
// ConventionService conventionService,
// BuilderFactory builderFactory) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(ReferenceSearchBuilder.class)));
// this.builderFactory = builderFactory;
// }
//
// public ReferenceSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Reference> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Reference> models = new ArrayList<>();
// for (Map<String, String> d : data) {
// Reference m = new Reference();
// if (d.containsKey("id")) m.setId(UUID.fromString(d.getOrDefault("id", null)));
// m.setLabel(d.getOrDefault("name", null));
// m.setSourceType(ReferenceSourceType.External);
// String type = d.getOrDefault("referenceType", null);
// if (type != null) m.setType(ReferenceType.valueOf(type));
//
// Definition definition = this.builderFactory.builder(DefinitionSearchBuilder.class).authorize(this.authorize).build(null, d);
// m.setDefinition(definition);
//
// switch (m.getType()){
// case Researcher:
// case Organizations:
// case Funder:
// case Grants:
// case DataRepositories:
// case PubRepositories:
// case Journals:
// case Datasets:
// case Registries:
// case Services:{
// for (Field field : definition.getFields()) {
// if (field.getCode().equals("pid") && field.getValue() != null) {
// if (d.containsKey("key")) {
// m.setReference(d.get("key") + ":" + field.getValue());
// }
// }
// if (field.getCode().equals("tag") && field.getValue() != null){
// m.setSource(field.getValue());
// }
// }
// }
// }
//
// m.setDescription(d.getOrDefault("description", null));
// m.setAbbreviation(d.getOrDefault("abbreviation", null));
//
// models.add(m);
//
// }
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//
//}

8
dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java
View File

@ -20,6 +20,7 @@ import eu.eudat.model.descriptionproperties.PropertyDefinitionFieldSetItem;
import eu.eudat.model.persist.DescriptionProfilingRequest;
import eu.eudat.model.persist.DescriptionProfilingWithDataRequest;
import eu.eudat.service.remotefetcher.ExternalUrlConfigProvider;
import eu.eudat.service.remotefetcher.RemoteFetcherService;
import eu.eudat.service.remotefetcher.config.entities.*;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import gr.cite.tools.data.builder.BuilderFactory;
@ -66,6 +67,7 @@ public class PrefillingServiceImpl implements PrefillingService {
private final ExternalUrlConfigProvider externalUrlConfigProvider;
private final XmlHandlingService xmlHandlingService;
private final ValidatorFactory validatorFactory;
private final RemoteFetcherService remoteFetcherService;
@Autowired
public PrefillingServiceImpl(
EntityManager entityManager,
@ -76,7 +78,8 @@ public class PrefillingServiceImpl implements PrefillingService {
JsonHandlingService jsonHandlingService,
ExternalUrlConfigProvider externalUrlConfigProvider,
XmlHandlingService xmlHandlingService,
ValidatorFactory validatorFactory) {
ValidatorFactory validatorFactory,
RemoteFetcherService remoteFetcherService) {
this.entityManager = entityManager;
this.builderFactory = builderFactory;
this.queryFactory = queryFactory;
@ -86,6 +89,7 @@ public class PrefillingServiceImpl implements PrefillingService {
this.externalUrlConfigProvider = externalUrlConfigProvider;
this.xmlHandlingService = xmlHandlingService;
this.validatorFactory = validatorFactory;
this.remoteFetcherService = remoteFetcherService;
}
@Override
@ -98,7 +102,7 @@ public class PrefillingServiceImpl implements PrefillingService {
List<Map<String, String>> map;
Map<String, PrefillingConfig> prefillingConfigs = this.externalUrlConfigProvider.getExternalUrls().getPrefillings();
// for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {//TODO new reference logic
// map = remoteFetcherService.getExternalGeneric(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());
// map = remoteFetcherService.getExternalData(prefillingConfig.getPrefillingSearch().getUrls(), externalReferenceCriteria, prefillingConfig.getPrefillingSearch().getFetchMode());
// prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList());
// if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) {
// List<Map<String, Object>> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());

6
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java
View File

@ -2,10 +2,12 @@ package eu.eudat.service.reference;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.Reference;
import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException;
@ -24,5 +26,5 @@ public interface ReferenceService {
void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException;
List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException;
List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException;
}

713
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java
View File

@ -1,18 +1,12 @@
package eu.eudat.service.reference;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.commons.types.referencetype.*;
@ -20,17 +14,18 @@ import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.Reference;
import eu.eudat.model.ReferenceType;
import eu.eudat.model.builder.ReferenceBuilder;
import eu.eudat.model.deleter.ReferenceDeleter;
import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.model.persist.referencedefinition.DefinitionPersist;
import eu.eudat.model.persist.referencedefinition.FieldPersist;
import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.ReferenceTypeQuery;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
import eu.eudat.service.remotefetcher.RemoteFetcherService;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory;
@ -45,28 +40,17 @@ import gr.cite.tools.logging.LoggerService;
import gr.cite.tools.logging.MapLogEntry;
import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import net.minidev.json.JSONArray;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import javax.management.InvalidApplicationException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class ReferenceServiceImpl implements ReferenceService {
@ -81,8 +65,7 @@ public class ReferenceServiceImpl implements ReferenceService {
private final QueryFactory queryFactory;
private final XmlHandlingService xmlHandlingService;
private final WebClient client;
public final RemoteFetcherService remoteFetcherService;
public ReferenceServiceImpl(
EntityManager entityManager,
AuthorizationService authorizationService,
@ -91,7 +74,7 @@ public class ReferenceServiceImpl implements ReferenceService {
ConventionService conventionService,
MessageSource messageSource,
QueryFactory queryFactory,
XmlHandlingService xmlHandlingService) {
XmlHandlingService xmlHandlingService, RemoteFetcherService remoteFetcherService) {
this.entityManager = entityManager;
this.authorizationService = authorizationService;
this.deleterFactory = deleterFactory;
@ -100,11 +83,7 @@ public class ReferenceServiceImpl implements ReferenceService {
this.messageSource = messageSource;
this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService;
this.client = WebClient.builder().codecs(clientCodecConfigurer -> {
clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
}
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
this.remoteFetcherService = remoteFetcherService;
}
@Override
@ -178,380 +157,340 @@ public class ReferenceServiceImpl implements ReferenceService {
this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id));
}
// public List<FetcherReference> searchReference(ReferenceType externalType, String query, String type) throws HugeResultSet, MyNotFoundException, InvalidApplicationException {
// ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
//
// List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(externalType, externalUrlCriteria, type);
//
// List<FetcherReference> list = this.fetchFromDb(externalType, query, type, remoteRepos);
//
// list.addAll(remoteRepos.stream().map(FetcherReference::fromRemoteModel).toList());
// list = list.stream().filter(x -> x.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
// list.sort(Comparator.comparing(FetcherReference::getName));
// return list;
// }
@Override
public List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException {
int initialOffset = 0;
if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
initialOffset = lookup.getPage().getOffset();
lookup.getPage().setOffset(0);
}
// @Override
// public List<Reference> searchReference(ReferenceSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
// int initialOffset = 0;
// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
// initialOffset = lookup.getPage().getOffset();
// lookup.getPage().setOffset(0);
// }
//
// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
//
// List<Map<String, String>> remoteRepos = remoteFetcherService.getReferences(lookup.getType(), externalReferenceCriteria, lookup.getKey());
//
// List<Reference> externalModels = this.builderFactory.builder(ReferenceSearchBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), remoteRepos);
// List<Reference> models = this.fetchFromDb(lookup);
//
// models.addAll(externalModels);
//
// if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).collect(Collectors.toList()); }
// models.sort(Comparator.comparing(Reference::getLabel));
//
// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
// models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList();
// }
//
// return models;
// }
ReferenceTypeEntity data = this.entityManager.find(ReferenceTypeEntity.class, lookup.getTypeId());
if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{lookup.getTypeId(), ReferenceType.class.getSimpleName()}, LocaleContextHolder.getLocale()));
private List<Reference> fetchFromDb(ReferenceSearchLookup lookup){
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal);
ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
ExternalDataResult remoteRepos = this.getReferenceData(data, externalReferenceCriteria, lookup.getKey());
List<Reference> externalModels = new ArrayList<>();
if (remoteRepos != null && this.conventionService.isListNullOrEmpty(remoteRepos.getResults())){
List<ReferenceEntity> referenceEntities = new ArrayList<>();
for (Map<String, String> result : remoteRepos.getResults()){
if (result == null || result.isEmpty()) continue;;
ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data, remoteRepos);
referenceEntities.add(referenceEntity);
}
externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities);
}
List<Reference> models = this.fetchReferenceFromDb(lookup);
for (Reference externalReference : externalModels){
if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null &&x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference);
}
if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).toList(); }
models.sort(Comparator.comparing(Reference::getLabel));
if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList();
}
return models;
}
@NotNull
private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) {
ReferenceEntity referenceEntity = new ReferenceEntity(); //TODO new reference logic
referenceEntity.setTypeId(data.getId());
referenceEntity.setIsActive(IsActive.Active);
referenceEntity.setReference(result.getOrDefault("key", null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("pid", null));
referenceEntity.setSource(result.getOrDefault("tag", null));
referenceEntity.setAbbreviation(result.getOrDefault("abbreviation", null));
referenceEntity.setDescription(result.getOrDefault("description", null));
referenceEntity.setLabel(result.getOrDefault("name", null));
referenceEntity.setSourceType(ReferenceSourceType.External);
DefinitionEntity definitionEntity = new DefinitionEntity();
definitionEntity.setFields(new ArrayList<>());
for (Map.Entry<String, String> resultValue : result.entrySet()){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode(resultValue.getKey());
fieldEntity.setValue(resultValue.getValue());
fieldEntity.setDataType(ReferenceFieldDataType.Text);
definitionEntity.getFields().add(fieldEntity);
}
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
return referenceEntity;
}
private List<Reference> fetchReferenceFromDb(ReferenceSearchLookup lookup){
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId());
List<ReferenceEntity> data = query.collectAs(lookup.getProject());
return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data);
}
// private List<FetcherReference> fetchFromDb(ReferenceType externalType, String query, String type, List<Map<String, String>> remoteRepos) throws InvalidApplicationException {
// List<FetcherReference> list = new LinkedList<>();
// switch (externalType) {
// case DataRepositories:
// case PubRepositories:
// case Journals: {
// DataRepositoryCriteria criteria = new DataRepositoryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<DataRepository> dataRepositoryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao().getWithCriteria(criteria)).toList();
// list = dataRepositoryList.stream().map(item -> new FetcherReference().fromDataRepository(item)).collect(Collectors.toList());
// }
// }
// case Registries: {
// RegistryCriteria criteria = new RegistryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<Registry> registryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao().getWithCriteria(criteria)).toList();
// list = registryList.stream().map(item -> new FetcherReference().fromRegistry(item)).collect(Collectors.toList());
// }
// }
// case Services:
// {
// ServiceCriteria criteria = new ServiceCriteria();
//
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
//
// if (type.equals("")) {
// List<Service> serviceList = (this.apiContext.getOperationsContext().getDatabaseRepository().getServiceDao().getWithCriteria(criteria)).toList();
// list = serviceList.stream().map(item -> new FetcherReference().fromService(item)).collect(Collectors.toList());
// }
// }
// case Datasets:{
//
// ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build();
//
// criteria.setCreationUserId(this.userScope.getUserId());
// QueryableList<ExternalDataset> items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria);
//
// list = items.select(item -> new FetcherReference().fromDataset(item));
// }
// case Taxonomies:
// case Publications:
// case Licenses:
// break;
// }
//
// return list;
// }
private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) {
ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition());
if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult();
@Override
public List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
ReferenceTypeQuery query = this.queryFactory.query(ReferenceTypeQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(lookup.getReferenceTypeId());
List<ReferenceTypeEntity> datas = query.collectAs(lookup.getProject());
if (datas.size() != 1 ){
return null;
ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).toList(), externalReferenceCriteria, key, null);
for (Map<String, String> result: results.getResults()) {
result.put("referenceType", referenceType.getName());
}
ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, datas.get(0).getDefinition());
List<Map<String, String>> remoteRepos = this.getAll(definition.getSources(), lookup);
return null;
}
private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
List<Map<String, String>> results = new LinkedList<>();
if (sources == null || sources.isEmpty()) {
return results;
}
sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
// private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
// List<Map<String, String>> results = new LinkedList<>();
//
// if (sources == null || sources.isEmpty()) {
// return results;
// }
//
// sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
//
// List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
// apiSources.forEach(source -> {
// try {
// String auth = null;
// if (source.getAuth()!= null) {
// //auth = this.getAuthentication(source.getAuth());
// }
// results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// });
// List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
// staticSources.forEach(source -> {
// Map<String, String> map = new HashMap<>();
// source.getOptions().forEach(option -> {
// map.put(option.getCode(), option.getValue());
// map.put("tag", source.getLabel());
// map.put("key", source.getKey());
// });
// results.add(map);
// });
//
// return results;
// }
List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
apiSources.forEach(source -> {
try {
String auth = null;
if (source.getAuth()!= null) {
//auth = this.getAuthentication(source.getAuth());
}
results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
});
List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
staticSources.forEach(source -> {
Map<String, String> map = new HashMap<>();
source.getOptions().forEach(option -> {
map.put(option.getCode(), option.getValue());
map.put("tag", source.getLabel());
map.put("key", source.getKey());
});
results.add(map);
});
// private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
// Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
// .contentType(MediaType.APPLICATION_JSON)
// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
// })).block();
//
//
//
// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
// }
return results;
}
// private String parseBodyString(String bodyString) {
// String finalBodyString = bodyString;
// if (bodyString.contains("{env:")) {
// int index = bodyString.indexOf("{env: ");
// while (index >= 0) {
// int endIndex = bodyString.indexOf("}", index + 6);
// String envName = bodyString.substring(index + 6, endIndex);
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// index = bodyString.indexOf("{env: ", index + 6);
// }
// }
// return finalBodyString;
// }
private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
.contentType(MediaType.APPLICATION_JSON)
.bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
.exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
})).block();
// private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
// Set<Integer> pages = new HashSet<>();
//
// String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
// String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
//
// ExternalDataResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
// if(externalRefernceResult != null) {
// if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
// .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
// .collect(Collectors.toList()));
// }
// if (fetchStrategy == FetchStrategy.FIRST)
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
//
// //Long maxResults = configLoader.getExternalUrls().getMaxresults();
// Long maxResults = Long.valueOf(1000);
// if (externalRefernceResult.getResults().size() > maxResults)
// throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getResults().size() + " results... Please submit a more detailed search query");
//
// Optional<ExternalDataResult> optionalResults = pages.parallelStream()
// .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
// .filter(Objects::nonNull)
// .reduce((result1, result2) -> {
// result1.getResults().addAll(result2.getResults());
// return result1;
// });
// ExternalDataResult remainingExternalDataResult = optionalResults.orElseGet(ExternalDataResult::new);
// remainingExternalDataResult.getResults().addAll(externalRefernceResult.getResults());
//
// return remainingExternalDataResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
// }
// else {
// return new LinkedList<>();
// }
// }
// private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
// String completedPath = urlPath;
//
// if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
// if (lookup.getLike() != null) {
// completedPath = completedPath.replace("{like}", lookup.getLike());
// } else {
// completedPath = completedPath.replace("{like}", "*");
// }
// }
//
// if (urlPath.contains("{like}")){
// if (lookup.getLike() != null) {
// completedPath = completedPath.replace("{like}", lookup.getLike());
// } else {
// completedPath = completedPath.replace("{like}", "");
// }
// }
//
// if (urlPath.contains("{page}")) {
// if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
// completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
// } else if (firstPage != null) {
// completedPath = completedPath.replace("{page}", firstPage);
// } else {
// completedPath = completedPath.replace("{page}", "1");
// }
// }
//
// if (urlPath.contains("{pageSize}")){
// if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
// completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
// } else {
// completedPath = completedPath.replace("{pageSize}", "100");
// }
// }
//
//
// return completedPath;
// }
// protected ExternalDataResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
//
// try {
// ResponseEntity<String> response;
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
//
// response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
// if (contentType != null && !contentType.isEmpty()) {
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
// httpHeaders.setContentType(MediaType.valueOf(contentType));
// }
// if (auth != null) {
// httpHeaders.set("Authorization", auth);
// }
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
// if (response.getStatusCode() == HttpStatus.OK) { // success
// ExternalDataResult externalDataResult = new ExternalDataResult();
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
// externalDataResult = this.getFromJson(jsonContext, resultsEntity);
// }
//
// return externalDataResult;
// }
// } catch (Exception exception) {
// logger.error(exception.getMessage(), exception);
// }
//
// return null;
// }
return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
}
private String parseBodyString(String bodyString) {
String finalBodyString = bodyString;
if (bodyString.contains("{env:")) {
int index = bodyString.indexOf("{env: ");
while (index >= 0) {
int endIndex = bodyString.indexOf("}", index + 6);
String envName = bodyString.substring(index + 6, endIndex);
finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
index = bodyString.indexOf("{env: ", index + 6);
}
}
return finalBodyString;
}
private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
Set<Integer> pages = new HashSet<>();
String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
if(externalRefernceResult != null) {
if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
if (fetchStrategy == FetchStrategy.FIRST)
return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set
for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++)
pages.add(i);
//Long maxResults = configLoader.getExternalUrls().getMaxresults();
Long maxResults = Long.valueOf(1000);
if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults))
throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query");
Optional<ExternalRefernceResult> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
.filter(Objects::nonNull)
.reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults());
return result1;
});
ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults());
return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
}
else {
return new LinkedList<>();
}
}
private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
String completedPath = urlPath;
if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
if (lookup.getLike() != null) {
completedPath = completedPath.replace("{like}", lookup.getLike());
} else {
completedPath = completedPath.replace("{like}", "*");
}
}
if (urlPath.contains("{like}")){
if (lookup.getLike() != null) {
completedPath = completedPath.replace("{like}", lookup.getLike());
} else {
completedPath = completedPath.replace("{like}", "");
}
}
if (urlPath.contains("{page}")) {
if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
} else if (firstPage != null) {
completedPath = completedPath.replace("{page}", firstPage);
} else {
completedPath = completedPath.replace("{page}", "1");
}
}
if (urlPath.contains("{pageSize}")){
if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
} else {
completedPath = completedPath.replace("{pageSize}", "100");
}
}
// public static ExternalDataResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
// return new ExternalDataResult(parseData(jsonContext, resultsEntity));
// }
return completedPath;
}
protected ExternalRefernceResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
try {
ResponseEntity<String> response;
JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
if (contentType != null && !contentType.isEmpty()) {
httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
httpHeaders.setContentType(MediaType.valueOf(contentType));
}
if (auth != null) {
httpHeaders.set("Authorization", auth);
}
}).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
if (response.getStatusCode() == HttpStatus.OK) { // success
ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult();
if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
DocumentContext jsonContext = JsonPath.parse(response.getBody());
externalRefernceResult = this.getFromJson(jsonContext, resultsEntity);
}
if (externalRefernceResult.getPagination().size() == 0) {
externalRefernceResult.getPagination().put("pages", 1);
externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size());
}
return externalRefernceResult;
}
} catch (Exception exception) {
logger.error(exception.getMessage(), exception);
}
return null;
}
public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
return new ExternalRefernceResult(parseData(jsonContext, resultsEntity),
new HashMap<>(1, 1));
}
private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
List<Map<String, String>> parsedData = new ArrayList<>();
for (Map<String, String> stringObjectMap: rawData){
Map<String, String> map = new HashMap<>();
for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
String pathValue = field.getResponsePath();
if (!pathValue.contains(".")){
if (stringObjectMap.containsKey(pathValue)) {
//map.put(field.getCode(), stringObjectMap.get(pathValue));
map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
}
}else {
if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
String value = null;
Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
if (fieldObj != null){
if (fieldObj instanceof Map){
Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
} else if (fieldObj instanceof List) {
Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
}
}
if (value != null){
map.put(field.getCode(), value);
}
}
}
}
parsedData.add(map);
}
return parsedData;
}
private static String normalizeValue(Object value) {
if (value instanceof JSONArray) {
JSONArray jarr = (JSONArray) value;
if (jarr.get(0) instanceof String) {
return jarr.get(0).toString();
} else {
for (Object o : jarr) {
if ((o instanceof Map) && ((Map) o).containsKey("content")) {
try {
return ((Map<String, String>) o).get("content");
}
catch (ClassCastException e){
if(((Map<?, ?>) o).get("content") instanceof Integer) {
return String.valueOf(((Map<?, ?>) o).get("content"));
}
return null;
}
}
}
}
} else if (value instanceof Map) {
String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
return ((Map<String, String>)value).get(key);
}
return value != null ? value.toString() : null;
}
// private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
// List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
// List<Map<String, String>> parsedData = new ArrayList<>();
//
// for (Map<String, String> stringObjectMap: rawData){
// Map<String, String> map = new HashMap<>();
// for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
// String pathValue = field.getResponsePath();
// if (!pathValue.contains(".")){
// if (stringObjectMap.containsKey(pathValue)) {
// //map.put(field.getCode(), stringObjectMap.get(pathValue));
// map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
// }
// }else {
// if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
// String value = null;
// Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
// if (fieldObj != null){
// if (fieldObj instanceof Map){
// Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
// if(o instanceof String){
// value = (String)o;
// }
// else if(o instanceof Integer){
// value = String.valueOf(o);
// }
// } else if (fieldObj instanceof List) {
// Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
// if(o instanceof String){
// value = (String)o;
// }
// else if(o instanceof Integer){
// value = String.valueOf(o);
// }
// }
// }
// if (value != null){
// map.put(field.getCode(), value);
// }
// }
// }
// }
// parsedData.add(map);
// }
// return parsedData;
// }
//
//
// private static String normalizeValue(Object value) {
// if (value instanceof JSONArray) {
// JSONArray jarr = (JSONArray) value;
// if (jarr.get(0) instanceof String) {
// return jarr.get(0).toString();
// } else {
// for (Object o : jarr) {
// if ((o instanceof Map) && ((Map) o).containsKey("content")) {
// try {
// return ((Map<String, String>) o).get("content");
// }
// catch (ClassCastException e){
// if(((Map<?, ?>) o).get("content") instanceof Integer) {
// return String.valueOf(((Map<?, ?>) o).get("content"));
// }
// return null;
// }
// }
// }
// }
// } else if (value instanceof Map) {
// String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
// return ((Map<String, String>)value).get(key);
// }
// return value != null ? value.toString() : null;
// }
}

1
dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java
View File

@ -18,7 +18,6 @@ import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException;

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java
View File

@ -1,21 +1,14 @@
//package eu.eudat.service.remotefetcher;
//
//import eu.eudat.commons.exceptions.HugeResultSetException;
//import eu.eudat.service.remotefetcher.config.entities.GenericUrls;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import gr.cite.tools.exception.MyNotFoundException;
//
//import java.util.List;
//import java.util.Map;
//
//public interface RemoteFetcherService {
// List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException;
//
// GenericUrls getExternalUrls(ReferenceType referenceType);
//
// Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException;
//
// List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
//
// List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
//}
package eu.eudat.service.remotefetcher;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import java.util.List;
public interface RemoteFetcherService {
ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy);
Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
}

913
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java
View File

@ -1,533 +1,388 @@
//package eu.eudat.service.remotefetcher;
//
//import com.fasterxml.jackson.core.type.TypeReference;
//import com.fasterxml.jackson.databind.JsonNode;
//import com.fasterxml.jackson.databind.ObjectMapper;
//import com.jayway.jsonpath.DocumentContext;
//import com.jayway.jsonpath.JsonPath;
//import eu.eudat.commons.exceptions.HugeResultSetException;
//import eu.eudat.service.remotefetcher.config.AuthenticationConfiguration;
//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
//import eu.eudat.service.remotefetcher.config.QueryConfig;
//import eu.eudat.service.remotefetcher.config.UrlConfiguration;
//import eu.eudat.service.remotefetcher.config.entities.GenericUrls;
//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
//import gr.cite.tools.exception.MyNotFoundException;
//import jakarta.xml.bind.JAXBContext;
//import jakarta.xml.bind.Unmarshaller;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.core.ParameterizedTypeReference;
//import org.springframework.http.*;
//import org.springframework.http.client.reactive.ReactorClientHttpConnector;
//import org.springframework.http.codec.json.Jackson2JsonDecoder;
//import org.springframework.stereotype.Service;
//import org.springframework.web.client.RestTemplate;
//import org.springframework.web.reactive.function.client.WebClient;
//import reactor.netty.http.client.HttpClient;
//
//import java.io.File;
//import java.io.StringReader;
//import java.lang.reflect.Method;
//import java.nio.file.Paths;
//import java.util.*;
//import java.util.stream.Collectors;
//
//@Service
//public class RemoteFetcherServiceImpl implements RemoteFetcherService {
// private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
//
// private WebClient webClient;
// private final ExternalUrlConfigProvider externalUrlConfigProvider;
// @Autowired
// public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider) {
// this.externalUrlConfigProvider = externalUrlConfigProvider;
// }
//
// private WebClient getWebClient(){
// if (this.webClient == null) {
// this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> {
// clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
// clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
// }
// ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
// }
// return webClient;
// }
//
// @Override
// public List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException {
// FetchStrategy fetchStrategy = null;
//
//// GenericUrls exGenericUrls = this.getExternalUrls(referenceType);
// List<UrlConfiguration> urlConfigs = key != null && !key.isEmpty() ? exGenericUrls.getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
// : exGenericUrls.getUrls();
//
// List<Map<String, String>> results = getAll(urlConfigs, fetchStrategy, externalReferenceCriteria);
// for (Map<String, String> result: results) {
// result.put("referenceType", referenceType.name());
// }
// return results;
// }
//
// @Override
// public GenericUrls getExternalUrls(ReferenceType referenceType) {
// return switch (referenceType) {
// case Taxonomies -> this.externalUrlConfigProvider.getExternalUrls().getTaxonomies();
// case Licenses -> this.externalUrlConfigProvider.getExternalUrls().getLicenses();
// case Publications -> this.externalUrlConfigProvider.getExternalUrls().getPublications();
// case Journals -> this.externalUrlConfigProvider.getExternalUrls().getJournals();
// case PubRepositories -> this.externalUrlConfigProvider.getExternalUrls().getPubRepositories();
// case DataRepositories -> this.externalUrlConfigProvider.getExternalUrls().getRepositories();
// case Registries -> this.externalUrlConfigProvider.getExternalUrls().getRegistries();
// case Services -> this.externalUrlConfigProvider.getExternalUrls().getServices();
// case Grants -> this.externalUrlConfigProvider.getExternalUrls().getGrants();
// case Organizations -> this.externalUrlConfigProvider.getExternalUrls().getOrganisations();
// case Datasets -> this.externalUrlConfigProvider.getExternalUrls().getDatasets();
// case Funder -> this.externalUrlConfigProvider.getExternalUrls().getFunders();
// case Project -> this.externalUrlConfigProvider.getExternalUrls().getProjects();
// case Researcher -> this.externalUrlConfigProvider.getExternalUrls().getResearchers();
// default -> throw new IllegalArgumentException("Type not found" + referenceType);
// };
// }
//
// @Override
// public Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException {
// List<UrlConfiguration> urlConfigs =
// key != null && !key.isEmpty() ? this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
// : this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls();
// FetchStrategy fetchStrategy = this.externalUrlConfigProvider.getExternalUrls().getValidations().getFetchMode();
// List<Map<String, String>> data = this.getAll(urlConfigs, fetchStrategy, externalReferenceCriteria);
// return data.size();
// }
//
// @Override
// public List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) {
// List<UrlConfiguration> urlConfigurations = genericUrls.getUrls();
// FetchStrategy fetchStrategy = genericUrls.getFetchMode();
// return getAll(urlConfigurations, fetchStrategy, externalReferenceCriteria);
// }
//
// @Override
// public List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) {
// List<UrlConfiguration> urlConfigurations = genericUrls.getUrls();
// return getAllWithData(urlConfigurations, externalReferenceCriteria);
// }
//
// private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
//
// List<Map<String, String>> results = new LinkedList<>();
//
// if (urlConfigs == null || urlConfigs.isEmpty()) {
// return results;
// }
//// throw new MyNotFoundException("No Repository urls found in configuration");
//
// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
// for (UrlConfiguration urlConfiguration : urlConfigs) {
// applyFunderQuery(urlConfiguration, externalReferenceCriteria);
// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
// try {
// String auth = null;
// if (urlConfiguration.getAuth() != null) {
// auth = this.getAuthentication(urlConfiguration.getAuth());
// }
// results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalReferenceCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType(), urlConfiguration.getQueries(), auth));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// } else if (urlConfiguration.getType() != null && urlConfiguration.getType().equals("Internal")) {
// results.addAll(getAllResultsFromMockUpJson(urlConfiguration.getUrl(), externalReferenceCriteria.getLike()));
// }
// }
// /* for (UrlConfiguration urlConfig : urlConfigs) {
// ifFunderQueryExist(urlConfig, externalUrlCriteria);
// if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
// results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType()));
// } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
// results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
// }
// }*/
// return results;
// }
//
// private String getAuthentication(AuthenticationConfiguration authenticationConfiguration) {
// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod());
// Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
// .contentType(MediaType.APPLICATION_JSON)
// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
// })).block();
//
//
//
// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
// }
//
// private List<Map<String, Object>> getAllWithData(List<UrlConfiguration> urlConfigs, ExternalReferenceCriteria externalReferenceCriteria) {
//
// List<Map<String, Object>> results = new LinkedList<>();
//
// if (urlConfigs == null || urlConfigs.isEmpty()) {
// return results;
// }
//
// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
// urlConfigs.forEach(urlConfiguration -> {
// applyFunderQuery(urlConfiguration, externalReferenceCriteria);
// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
// try {
// results.addAll(getAllResultsFromUrlWithData(urlConfiguration.getUrl(), urlConfiguration.getData(), externalReferenceCriteria, urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getQueries()));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// }
// });
// return results;
//
// }
//
// private void applyFunderQuery(UrlConfiguration urlConfiguration, ExternalReferenceCriteria externalReferenceCriteria) {
// if (urlConfiguration.getFunderQuery() != null) {
// if (externalReferenceCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) {
// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery()));
package eu.eudat.service.remotefetcher;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.convention.ConventionService;
import eu.eudat.service.remotefetcher.config.entities.*;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import gr.cite.tools.exception.MyApplicationException;
import net.minidev.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.*;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import java.io.File;
import java.nio.file.Paths;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class RemoteFetcherServiceImpl implements RemoteFetcherService {
private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
private WebClient webClient;
private final ExternalUrlConfigProvider externalUrlConfigProvider;
private final ConventionService conventionService;
private final XmlHandlingService xmlHandlingService;
@Autowired
public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) {
this.externalUrlConfigProvider = externalUrlConfigProvider;
this.conventionService = conventionService;
this.xmlHandlingService = xmlHandlingService;
}
private WebClient getWebClient(){
if (this.webClient == null) {
this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> {
clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
}
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
}
return webClient;
}
@Override
public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) {
List<SourceBaseConfiguration> apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList();
if (!this.conventionService.isNullOrEmpty(key)){
apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList();
}
if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult();
apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal));
return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria);
}
@Override
public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size();
}
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
ExternalDataResult results = new ExternalDataResult();
if (this.conventionService.isListNullOrEmpty(sources)) return new ExternalDataResult();
for (SourceBaseConfiguration source : sources) {
if (source.getType() == null || source.getType().equals(ReferenceTypeSourceType.API)) {
try {
SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource = (SourceExternalApiConfiguration)source;
this.applyFunderToQuery(apiSource, externalReferenceCriteria);
String auth = null;
if (apiSource.getAuth() != null) {
auth = this.buildAuthentication(apiSource.getAuth());
}
results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
} else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) {
SourceStaticOptionConfiguration<StaticOption> staticSource = (SourceStaticOptionConfiguration)source;
results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO:
}
}
return results;
}
private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) {
HttpMethod method;
switch (authenticationConfiguration.getAuthMethod()) {
case GET -> method = HttpMethod.GET;
case POST -> method =HttpMethod.POST;
default -> throw new MyApplicationException("unrecognized type " + authenticationConfiguration.getAuthMethod());
}
Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
.contentType(MediaType.APPLICATION_JSON)
.bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
.exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
})).block();
if (response == null) throw new MyApplicationException("Authentication " + authenticationConfiguration.getAuthUrl() + " failed");
return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null);
}
private void applyFunderToQuery(SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) {
//TODO new reference logic
// if (apiSource.getFunderQuery() != null) {
// if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) {
// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery()));
// }
// else {
// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", ""));
// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", ""));
// }
// }
// }
//
// private String calculateQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
// String finalQuery = "";
// QueryConfig queryConfig = queryConfigs.stream().filter(queryConfigl -> externalReferenceCriteria.getLike().matches(queryConfigl.getCondition()))
// .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
// if (queryConfig != null) {
// if (queryConfig.getSeparator() != null) {
// String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
// finalQuery = queryConfig.getValue();
// for (int i = 0; i < likes.length; i++) {
// finalQuery = finalQuery.replaceAll("\\{like" + (i+1) + "}", likes[i]);
// }
// } else {
// finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike());
// }
//
// }
// return finalQuery;
// }
//
// protected String replaceCriteriaOnUrl(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
// String completedPath = path;
// if (externalReferenceCriteria.getLike() != null) {
// if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().equals("")) {
// completedPath = completedPath.replaceAll("\\{like}", "*");
// completedPath = completedPath.replaceAll("\\{query}", "*");
// } else {
// if (completedPath.contains("{query}")) {
// completedPath = completedPath.replaceAll("\\{query}", this.calculateQuery(externalReferenceCriteria, queries));
// } else {
// completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike());
// }
// }
// } else {
// completedPath = completedPath.replace("{like}", "");
// }
// if (externalReferenceCriteria.getFunderId() != null) {
// String funderPrefix = externalReferenceCriteria.getFunderId().split(":")[0];
// String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", "");
// if (funderId.toCharArray()[0] == ':') {
// funderId = externalReferenceCriteria.getFunderId();
// }
// /*
// try { funderId = URLEncoder.encode(funderId, "UTF-8"); } catch
// (UnsupportedEncodingException e) { logger.error(e.getMessage(), e); }
// */
// completedPath = completedPath.replace("{funderId}", funderId);
// }
// else if(completedPath.contains("{funderId}")){
// logger.warn("FunderId is null.");
// completedPath = completedPath.replace("{funderId}", " ");
// }
// if (externalReferenceCriteria.getPage() != null) {
// completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
// } else {
// if (firstPage != null) {
// completedPath = completedPath.replace("{page}", firstPage);
// } else {
// completedPath = completedPath.replace("{page}", "1");
// }
// }
// if (externalReferenceCriteria.getPageSize() != null) {
// completedPath = completedPath.replace("{pageSize}", externalReferenceCriteria.getPageSize());
// } else {
// completedPath = completedPath.replace("{pageSize}", "60");
// }
// if (externalReferenceCriteria.getHost() != null) {
// completedPath = completedPath.replace("{host}", externalReferenceCriteria.getHost());
// } else {
// completedPath = completedPath.replace("{host}", "");
// }
// if (externalReferenceCriteria.getPath() != null) {
// completedPath = completedPath.replace("{path}", externalReferenceCriteria.getPath());
// } else {
// completedPath = completedPath.replace("{path}", "");
// }
// return completedPath;
// }
//
// private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalReferenceCriteria externalReferenceCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType, List<QueryConfig> queries, String auth) throws Exception {
// Set<Integer> pages = new HashSet<>();
//
// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries);
// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries);
//
// ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth);
// if(externalRefernceResult != null) {
// if (filterType != null && filterType.equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
// .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
// .collect(Collectors.toList()));
// }
// if (fetchStrategy == FetchStrategy.FIRST)
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
//
// if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set
// for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++)
// pages.add(i);
//
// Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
// if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults))
// throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query");
//
// Optional<ExternalRefernceResult> optionalResults = pages.parallelStream()
// .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth))
// .filter(Objects::nonNull)
// .reduce((result1, result2) -> {
// result1.getResults().addAll(result2.getResults());
// return result1;
// });
// ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
// remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults());
//
// return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
// }
// else {
// return new LinkedList<>();
// }
// }
//
// private List<Map<String, Object>> getAllResultsFromUrlWithData(String path, final DataUrlConfiguration jsonDataPath, ExternalReferenceCriteria externalReferenceCriteria, String contentType, String firstPage, String requestBody, String requestType, List<QueryConfig> queries) {
//
// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries);
// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries);
//
// try {
// RestTemplate restTemplate = new RestTemplate();
// HttpHeaders headers = new HttpHeaders();
// HttpEntity<JsonNode> entity;
// ResponseEntity<String> response;
// if (contentType != null && !contentType.isEmpty()) {
// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
// headers.setContentType(MediaType.valueOf(contentType));
// }
// JsonNode jsonBody = new ObjectMapper().readTree(replacedBody);
// entity = new HttpEntity<>(jsonBody, headers);
//
// response = restTemplate.exchange(replacedPath, HttpMethod.valueOf(requestType), entity, String.class);
// if (response.getStatusCode() == HttpStatus.OK) {
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
// return jsonContext.read(jsonDataPath.getPath());
// }
// }
// }
// catch (Exception exception) {
// logger.error(exception.getMessage(), exception);
// }
//
// return new LinkedList<>();
// }
//
//
// protected ExternalRefernceResult getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType, String auth) {
//
// try {
// //RestTemplate restTemplate = new RestTemplate(new SimpleClientHttpRequestFactory());
// //HttpHeaders headers = new HttpHeaders();
// //HttpEntity<JsonNode> entity;
// ResponseEntity<String> response;
// /*
// * URL url = new URL(urlString.replaceAll(" ", "%20"));
// *
// * HttpURLConnection con = (HttpURLConnection) url.openConnection();
// * con.setRequestMethod("GET");
// */
// /* if (contentType != null && !contentType.isEmpty()) {
// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
// headers.setContentType(MediaType.valueOf(contentType));
// }
// if (auth != null) {
// headers.set("Authorization", auth);
// }*/
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
//// entity = new HttpEntity<>(jsonBody, headers);
//
//
// response = this.getWebClient().method(HttpMethod.valueOf(requestType)).uri(urlString).headers(httpHeaders -> {
// if (contentType != null && !contentType.isEmpty()) {
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
// httpHeaders.setContentType(MediaType.valueOf(contentType));
// }
// if (auth != null) {
// httpHeaders.set("Authorization", auth);
// }
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
// //response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class);
// if (response.getStatusCode() == HttpStatus.OK) { // success
// //do here all the parsing
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult();
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
//
// if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType, auth);
// } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
// } else {
// externalRefernceResult = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath);
// }
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
// .collect(Collectors.toList()));
// }
// else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) {
// Class<?> aClass = Class.forName(jsonDataPath.getParseClass());
// JAXBContext jaxbContext = JAXBContext.newInstance(aClass);
// Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
// StringReader stringReader = new StringReader(response.getBody());
// Object data = unmarshaller.unmarshal(stringReader);
// Method reader = null;
// if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) {
// String camelCaseGetter = jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty() ? "get" + jsonDataPath.getParseField().substring(0, 1).toUpperCase() + jsonDataPath.getParseField().substring(1) : "";
// reader = aClass.getMethod(camelCaseGetter);
// }
// ObjectMapper objectMapper = new ObjectMapper();
// List<Map<String, String>> values = new ArrayList<>();
// int max = 1;
// if (reader != null) {
// Object invokedField = reader.invoke(data);
// if (invokedField instanceof Collection) {
// max = ((Collection) invokedField).size();
// }
// }
// for (int i = 0; i< max; i++) {
// Object value;
// if (reader != null) {
// Object invokedField = reader.invoke(data);
// if (invokedField instanceof Collection) {
// value = ((Collection) invokedField).toArray()[i];
// } else {
// value = invokedField;
// }
// } else {
// value = data;
// }
// Map<String, String> map = objectMapper.convertValue(value, Map.class);
// if (jsonDataPath.getMergedFields() != null && !jsonDataPath.getMergedFields().isEmpty() && jsonDataPath.getMergedFieldName() != null && !jsonDataPath.getMergedFieldName().isEmpty()) {
// Map<String, String> finalMap = new HashMap<>();
// for (Map.Entry<String, String> entry : map.entrySet()) {
// if (jsonDataPath.getMergedFields().contains(entry.getKey())) {
// if (!finalMap.containsKey(jsonDataPath.getMergedFieldName())) {
// finalMap.put(jsonDataPath.getMergedFieldName(), entry.getValue());
// } else {
// finalMap.put(jsonDataPath.getMergedFieldName(), finalMap.get(jsonDataPath.getMergedFieldName()) + " " + entry.getValue());
// }
// } else {
// finalMap.put(entry.getKey(), entry.getValue());
// }
// }
// values.add(finalMap);
// } else {
// values.add(map);
// }
// }
// externalRefernceResult = new ExternalRefernceResult(values, new HashMap<>(1, 1));
// }
//
// if (externalRefernceResult.getPagination().isEmpty()) {
// externalRefernceResult.getPagination().put("pages", 1);
// externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size());
// }
// return externalRefernceResult;
// }
// } catch (Exception exception) {
// logger.error(exception.getMessage(), exception);
// } //maybe print smth...
//
// return null;
// }
//
// private List<Map<String, String>> getAllResultsFromMockUpJson(String path, String query) {
// List<Map<String, String>> internalResults;
// try {
// String filePath = Paths.get(path).toUri().toURL().toString();
// ObjectMapper mapper = new ObjectMapper();
// internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
// return searchListMap(internalResults, query);
// } catch (Exception e) {
// logger.error(e.getMessage(), e);
// return new LinkedList<>();
// }
// }
//
// private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
// List<Map<String, String>> list = new LinkedList<>();
// for (Map<String, String> map : internalResults)
// {
// if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
// list.add(map);
// }
// if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
// list.add(map);
// }
// }
// return list;
// }
//
// private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) {
// if(dataUrlConfiguration.getFieldsUrlConfiguration().getPid() == null)
// return "pid";
// else
// return "originalId";
// }
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPid() != null && key.equals("pid")) return "pid";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPidTypeField() != null && key.equals("pidTypeField")) return "pidTypeField";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPath() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getPath().replace("'",""))) return "path";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getHost() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getHost().replace("'",""))) return "host";
// return null;
// }
//
// private String parseBodyString(String bodyString) {
// String finalBodyString = bodyString;
// if (bodyString.contains("{env:")) {
// int index = bodyString.indexOf("{env: ");
// while (index >= 0) {
// int endIndex = bodyString.indexOf("}", index + 6);
// String envName = bodyString.substring(index + 6, endIndex);
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// index = bodyString.indexOf("{env: ", index + 6);
// }
// }
// return finalBodyString;
// }
//
//}
}
private String replaceLookupFieldQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
String finalQuery = "";
if (this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike())) return finalQuery;
QueryConfig queryConfig = queryConfigs.stream().filter(x -> !this.conventionService.isNullOrEmpty(x.getCondition()) && externalReferenceCriteria.getLike().matches(x.getCondition()))
.min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
if (queryConfig != null) {
if (queryConfig.getSeparator() != null) {
String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
finalQuery = queryConfig.getValue();
for (int i = 0; i < likes.length; i++) {
finalQuery = finalQuery.replaceAll("\\{like" + (i + 1) + "}", likes[i]);
}
} else {
finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike());
}
}
return finalQuery;
}
protected String replaceLookupFields(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
if (!this.conventionService.isNullOrEmpty(path)) return path;
String completedPath = path;
if (externalReferenceCriteria.getLike() != null) {
if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().isEmpty()) {
completedPath = completedPath.replaceAll("\\{like}", "*");
completedPath = completedPath.replaceAll("\\{query}", "*");
} else {
if (completedPath.contains("{query}")) {
completedPath = completedPath.replaceAll("\\{query}", this.replaceLookupFieldQuery(externalReferenceCriteria, queries));
} else {
completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike());
}
}
} else {
completedPath = completedPath.replace("{like}", "");
}
if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getFunderId())) {
String[] founderParts = externalReferenceCriteria.getFunderId().split(":");
String funderPrefix = founderParts.length > 0 ? founderParts[0] : "";
String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", "");
if (!funderId.isEmpty() && funderId.toCharArray()[0] == ':') {
funderId = externalReferenceCriteria.getFunderId();
}
completedPath = completedPath.replace("{funderId}", funderId);
} else if(completedPath.contains("{funderId}")){
logger.warn("FunderId is null.");
completedPath = completedPath.replace("{funderId}", " ");
}
if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPage())) completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
else if (!this.conventionService.isNullOrEmpty(firstPage)) completedPath = completedPath.replace("{page}", firstPage);
else completedPath = completedPath.replace("{page}", "1");
completedPath = completedPath.replace("{pageSize}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPageSize()) ? externalReferenceCriteria.getPageSize() : "60");
completedPath = completedPath.replace("{host}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getHost()) ? externalReferenceCriteria.getHost() : "");
completedPath = completedPath.replace("{path}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPath()) ? externalReferenceCriteria.getPath() : "");
return completedPath;
}
private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
String replacedPath = replaceLookupFields(apiSource.getUrl(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
String replacedBody = replaceLookupFields(apiSource.getRequestBody(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
if(externalDataResult != null) {
if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
externalDataResult.setResults(externalDataResult.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList());
if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
if (externalDataResult.getResults().size() > maxResults)
throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query");
return externalDataResult;
}
else {
return new ExternalDataResult();
}
}
protected ExternalDataResult getExternalDataResults(String urlString, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, String requestBody, String auth) {
try {
JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
HttpMethod method;
switch (apiSource.getHttpMethod()) {
case GET -> method = HttpMethod.GET;
case POST -> method =HttpMethod.POST;
default -> throw new MyApplicationException("unrecognized type " + apiSource.getHttpMethod());
}
ResponseEntity<String> response = this.getWebClient().method(method).uri(urlString).headers(httpHeaders -> {
if (this.conventionService.isNullOrEmpty(apiSource.getContentType())) {
httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(apiSource.getContentType())));
httpHeaders.setContentType(MediaType.valueOf(apiSource.getContentType()));
}
if (auth != null) {
httpHeaders.set("Authorization", auth);
}
}).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
if (response == null || !response.getStatusCode().isSameCodeAs(HttpStatus.OK) || !response.hasBody() || response.getBody() == null) return null;
//do here all the parsing
List<String> responseContentTypeHeader = response.getHeaders().getOrDefault("Content-Type", null);
String responseContentType = !this.conventionService.isListNullOrEmpty(responseContentTypeHeader) && responseContentTypeHeader.getFirst() != null ? responseContentTypeHeader.getFirst() : "";
if (responseContentType.contains("json") ) {
DocumentContext jsonContext = JsonPath.parse(response.getBody());
return this.parseData(jsonContext, apiSource.getResults());
} else {
throw new MyApplicationException("Unsupported response type" + responseContentType);
}
} catch (Exception exception) {
logger.error(exception.getMessage(), exception);
}
return null;
}
private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration<ResultFieldsMappingConfiguration> resultsConfigurationEntity) {
ExternalDataResult result = new ExternalDataResult();
if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult();
List<Map<String, Object>> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath());
result.setRawData(rawData);
if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult();
List<Map<String, String>> parsedData = new ArrayList<>();
for (Map<String, Object> stringObjectMap: rawData){
Map<String, String> map = new HashMap<>();
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){
String pathValue = field.getResponsePath();
if (!pathValue.contains(".")){
if (stringObjectMap.containsKey(pathValue)) {
map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
}
}else {
if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
String value = null;
Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
if (fieldObj != null){
if (fieldObj instanceof Map){
Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
} else if (fieldObj instanceof List) {
Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
}
}
if (value != null){
map.put(field.getCode(), value);
}
}
}
}
parsedData.add(map);
}
result.setResults(parsedData);
return result;
}
private static String normalizeValue(Object value) {
if (value instanceof JSONArray jsonArray) {
if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
return jsonArray.getFirst().toString();
} else {
for (Object o : jsonArray) {
if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
try {
return ((Map<String, String>) o).get("content");
}
catch (ClassCastException e){
if(((Map<?, ?>) o).get("content") instanceof Integer) {
return String.valueOf(((Map<?, ?>) o).get("content"));
}
return null;
}
}
}
}
} else if (value instanceof Map) {
String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
return ((Map<String, String>)value).get(key);
}
return value != null ? value.toString() : null;
}
private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) {
List<Map<String, String>> internalResults;
try {
String filePath = Paths.get(path).toUri().toURL().toString();
ObjectMapper mapper = new ObjectMapper();
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
return new ExternalDataResult(searchListMap(internalResults, query));
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new ExternalDataResult();
}
}
private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
List<Map<String, String>> list = new LinkedList<>();
for (Map<String, String> map : internalResults)
{
if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
list.add(map);
}
}
return list;
}
private String parseBodyString(String bodyString) {
String finalBodyString = bodyString;
if (bodyString.contains("{env:")) {
int index = bodyString.indexOf("{env: ");
while (index >= 0) {
int endIndex = bodyString.indexOf("}", index + 6);
String envName = bodyString.substring(index + 6, endIndex);
finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
index = bodyString.indexOf("{env: ", index + 6);
}
}
return finalBodyString;
}
}

152
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherUtils.java
View File

@ -1,152 +0,0 @@
//package eu.eudat.service.remotefetcher;
//
//import com.fasterxml.jackson.databind.ObjectMapper;
//import com.jayway.jsonpath.DocumentContext;
//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
//import net.minidev.json.JSONArray;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//
//import java.lang.reflect.InvocationTargetException;
//import java.lang.reflect.Method;
//import java.util.*;
//import java.util.stream.Collectors;
//
//public class RemoteFetcherUtils {
// private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class);
// private static final ObjectMapper mapper = new ObjectMapper();
//
// public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// return new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
// }
//
// public static ExternalRefernceResult getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcherServiceImpl remoteFetcherService, String requestBody, String requestType, String auth) {
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
//
// List<Map<String, String>> multiResults = externalRefernceResult.getResults().stream().map(result -> {
// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria();
// externalReferenceCriteria.setPath(result.get("path"));
// externalReferenceCriteria.setHost(result.get("host"));
// String replacedPath = remoteFetcherService.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalReferenceCriteria, jsonDataPath.getUrlConfiguration().getFirstpage(), jsonDataPath.getUrlConfiguration().getQueries());
// return remoteFetcherService.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType, auth);
// }).filter(Objects::nonNull).map(externalRefernceResult1 -> externalRefernceResult1.getResults().get(0)).collect(Collectors.toList());
// return new ExternalRefernceResult(multiResults, new HashMap<>(1, 1));
// }
//
// public static ExternalRefernceResult getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
// externalRefernceResult.getResults().stream().forEach(entry -> {
// String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
// entry.put("name", name);
// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
// });
// return externalRefernceResult;
// }
//
// private static List<Map<String, String>> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// List <Map<String, Object>> rawData = jsonContext.read(jsonDataPath.getPath());
// List<Map<String, String>> parsedData = new ArrayList<>();
// rawData.forEach(stringObjectMap -> {
// parsedData.add(new LinkedHashMap<>());
// Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> {
// String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1);
// Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0);
// try {
// String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration()));
// if (value != null) {
// if (field.getName().equals("pid") || field.getName().equals("pidTypeField")) {
// String pid = null;
// Object pidObj = stringObjectMap.get(value.split("\\.")[0]);
// if(pidObj != null){
// if(pidObj instanceof Map){
// Object o = ((Map<String, Object>) pidObj).get(value.split("\\.")[1]);
// if(o instanceof String){
// pid = (String)o;
// }
// else if(o instanceof Integer){
// pid = String.valueOf(o);
// }
// }
// else if(pidObj instanceof List){
// Object o = ((List<Map<String,?>>) pidObj).get(0).get(value.split("\\.")[1]);
// if(o instanceof String){
// pid = (String)o;
// }
// else if(o instanceof Integer){
// pid = String.valueOf(o);
// }
// }
// }
// if(pid != null) {
// if ((field.getName().equals("pid"))){
// parsedData.get(parsedData.size() - 1).put("pid", pid);
// }
// else{
// parsedData.get(parsedData.size() - 1).put("pidTypeField", pid);
// }
// }
// } else {
// value = value.replace("'", "");
// if (value.contains(".")) {
// String[] parts = value.split("\\.");
// Map<String, Object> tempMap = stringObjectMap;
// for (int i = 0; i < parts.length; i++) {
// if (tempMap.containsKey(parts[i])) {
// if (i + 1 < parts.length) {
// tempMap = (Map<String, Object>) tempMap.get(parts[i]);
// } else {
// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(tempMap.get(parts[i]), (field.getName().equals("types") || field.getName().equals("uri"))));
// }
// }
// }
// } else {
// if (stringObjectMap.containsKey(value)) {
// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri"))));
// }
// }
// }
// }
// } catch (IllegalAccessException | InvocationTargetException e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// });
// });
// return parsedData;
// }
//
// private static String normalizeValue(Object value, boolean jsonString) {
// if (value instanceof JSONArray) {
// if (jsonString) {
// return ((JSONArray)value).toJSONString();
// }
// JSONArray jarr = (JSONArray) value;
// if (jarr.get(0) instanceof String) {
// return jarr.get(0).toString();
// } else {
// for (Object o : jarr) {
// if ((o instanceof Map) && ((Map) o).containsKey("content")) {
// try {
// return ((Map<String, String>) o).get("content");
// }
// catch (ClassCastException e){
// if(((Map<?, ?>) o).get("content") instanceof Integer) {
// return String.valueOf(((Map<?, ?>) o).get("content"));
// }
// return null;
// }
// }
// }
// }
// } else if (value instanceof Map) {
// String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
// return ((Map<String, String>)value).get(key);
// }
// return value != null ? value.toString() : null;
// }
//}

57
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/AuthenticationConfiguration.java
View File

@ -1,57 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
public class AuthenticationConfiguration {
private String authUrl;
private String authMethod = "GET";
private String authTokenPath;
private String authRequestBody;
private String type;
public String getAuthUrl() {
return authUrl;
}
@XmlElement(name = "authUrl")
public void setAuthUrl(String authUrl) {
this.authUrl = authUrl;
}
public String getAuthMethod() {
return authMethod;
}
@XmlElement(name = "authUrlMethod")
public void setAuthMethod(String authMethod) {
this.authMethod = authMethod;
}
public String getAuthTokenPath() {
return authTokenPath;
}
@XmlElement(name = "authTokenJpath")
public void setAuthTokenPath(String authTokenPath) {
this.authTokenPath = authTokenPath;
}
public String getAuthRequestBody() {
return authRequestBody;
}
@XmlElement(name = "authUrlBody")
public void setAuthRequestBody(String authRequestBody) {
this.authRequestBody = authRequestBody;
}
public String getType() {
return type;
}
@XmlElement(name = "authType")
public void setType(String type) {
this.type = type;
}
}

134
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataFieldsUrlConfiguration.java
View File

@ -1,134 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
/**
* Created by ikalyvas on 6/29/2018.
*/
public class DataFieldsUrlConfiguration {
private String id;
private String name;
private String pid;
private String pidTypeField;
private String uri;
private String description;
private String source;
private String count;
private String path;
private String host;
private String types;
private String firstName;
private String lastName;
public String getId() {
return id;
}
@XmlElement(name = "id")
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
@XmlElement(name = "name")
public void setName(String name) {
this.name = name;
}
public String getPid() {
return pid;
}
@XmlElement(name = "pid")
public void setPid(String pid) {
this.pid = pid;
}
public String getPidTypeField() {
return pidTypeField;
}
@XmlElement(name = "pidTypeField")
public void setPidTypeField(String pidTypeField) {
this.pidTypeField = pidTypeField;
}
public String getUri() {
return uri;
}
@XmlElement(name = "uri")
public void setUri(String uri) {
this.uri = uri;
}
public String getDescription() {
return description;
}
@XmlElement(name = "description")
public void setDescription(String description) {
this.description = description;
}
public String getSource() {
return source;
}
@XmlElement(name = "source")
public void setSource(String source) {
this.source = source;
}
public String getCount() {
return count;
}
@XmlElement(name = "count")
public void setCount(String count) {
this.count = count;
}
public String getPath() {
return path;
}
@XmlElement(name = "path")
public void setPath(String path) {
this.path = path;
}
public String getHost() {
return host;
}
@XmlElement(name = "host")
public void setHost(String host) {
this.host = host;
}
@XmlElement(name = "types")
public String getTypes() {
return types;
}
public void setTypes(String types) {
this.types = types;
}
@XmlElement(name = "firstName")
public String getFirstName() {
return firstName;
}
public void setFirstName(String firstName) {
this.firstName = firstName;
}
@XmlElement(name = "lastName")
public String getLastName() {
return lastName;
}
public void setLastName(String lastName) {
this.lastName = lastName;
}
}

82
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataUrlConfiguration.java
View File

@ -1,82 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
/**
* Created by ikalyvas on 6/29/2018.
*/
public class DataUrlConfiguration {
private String path;
private DataFieldsUrlConfiguration fieldsUrlConfiguration;
private UrlConfiguration urlConfiguration;
private String parseClass;
private String parseField;
private List<String> mergedFields;
private String mergedFieldName;
public String getPath() {
return path;
}
@XmlElement(name = "path")
public void setPath(String path) {
this.path = path;
}
public DataFieldsUrlConfiguration getFieldsUrlConfiguration() {
return fieldsUrlConfiguration;
}
@XmlElement(name = "fields")
public void setFieldsUrlConfiguration(DataFieldsUrlConfiguration fieldsUrlConfiguration) {
this.fieldsUrlConfiguration = fieldsUrlConfiguration;
}
public UrlConfiguration getUrlConfiguration() {
return urlConfiguration;
}
@XmlElement(name = "urlConfig")
public void setUrlConfiguration(UrlConfiguration urlConfiguration) {
this.urlConfiguration = urlConfiguration;
}
public String getParseClass() {
return parseClass;
}
@XmlElement(name = "parse-class")
public void setParseClass(String parseClass) {
this.parseClass = parseClass;
}
public String getParseField() {
return parseField;
}
@XmlElement(name = "parse-field")
public void setParseField(String parseField) {
this.parseField = parseField;
}
public List<String> getMergedFields() {
return mergedFields;
}
@XmlElementWrapper(name = "merge-fields")
@XmlElement(name = "field")
public void setMergedFields(List<String> mergedFields) {
this.mergedFields = mergedFields;
}
public String getMergedFieldName() {
return mergedFieldName;
}
@XmlElement(name = "merge-field-name")
public void setMergedFieldName(String mergedFieldName) {
this.mergedFieldName = mergedFieldName;
}
}

173
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java
View File

@ -17,153 +17,8 @@ public class ExternalUrls implements Serializable {
Long maxresults;
GrantUrls grants;
ProjectUrls projects;
RegistryUrls registries;
RepositoryUrls repositories;
PubRepositoryUrls pubRepositories;
JournalUrls journals;
TaxonomiesUrls taxonomies;
PublicationsUrls publications;
ServiceUrls services;
ResearcherUrls researchers;
OrganisationUrls organisations;
DatasetUrls datasets;
/*TagUrls tags;*/
FunderUrls funders;
LicenseUrls licenses;
ValidationUrls validations;
Map<String, PrefillingConfig> prefillings;
public RegistryUrls getRegistries() {
return registries;
}
@XmlElement(name = "registries")
public void setRegistries(RegistryUrls registries) {
this.registries = registries;
}
public RepositoryUrls getRepositories() {
return repositories;
}
@XmlElement(name = "repositories")
public void setRepositories(RepositoryUrls repositories) {
this.repositories = repositories;
}
public PubRepositoryUrls getPubRepositories() {
return pubRepositories;
}
public void setPubRepositories(PubRepositoryUrls pubRepositories) {
this.pubRepositories = pubRepositories;
}
public void setJournals(JournalUrls journals) {
this.journals = journals;
}
public JournalUrls getJournals() {
return journals;
}
public JournalUrls getJournal() {
return journals;
}
@XmlElement(name = "journals")
public void setJournal(JournalUrls journal) {
this.journals = journal;
}
public TaxonomiesUrls getTaxonomies() {
return taxonomies;
}
@XmlElement(name = "taxonomies")
public void setTaxonomies(TaxonomiesUrls taxonomies) {
this.taxonomies = taxonomies;
}
public PublicationsUrls getPublications() {
return publications;
}
public void setPublications(PublicationsUrls publications) {
this.publications = publications;
}
public ServiceUrls getServices() {
return services;
}
@XmlElement(name = "services")
public void setServices(ServiceUrls services) {
this.services = services;
}
public ResearcherUrls getResearchers() {
return researchers;
}
@XmlElement(name = "researchers")
public void setResearchers(ResearcherUrls researchers) {
this.researchers = researchers;
}
/*public TagUrls getTags() {
return tags;
}
@XmlElement(name = "tags")
public void setTags(TagUrls tags) {
this.tags = tags;
}*/
public OrganisationUrls getOrganisations() {
return organisations;
}
@XmlElement(name = "organisations")
public void setOrganisations(OrganisationUrls organisations) {
this.organisations = organisations;
}
public GrantUrls getGrants() {
return grants;
}
@XmlElement(name = "grants")
public void setGrants(GrantUrls grants) {
this.grants = grants;
}
public ProjectUrls getProjects() {
return projects;
}
@XmlElement(name = "projects")
public void setProjects(ProjectUrls projects) {
this.projects = projects;
}
public FunderUrls getFunders() {
return funders;
}
public void setFunders(FunderUrls funders) {
this.funders = funders;
}
public Long getMaxresults() {
return maxresults;
}
@ -173,34 +28,6 @@ public class ExternalUrls implements Serializable {
this.maxresults = maxresults;
}
public DatasetUrls getDatasets() {
return datasets;
}
@XmlElement(name = "datasets")
public void setDatasets(DatasetUrls datasets) {
this.datasets = datasets;
}
public LicenseUrls getLicenses() {
return licenses;
}
@XmlElement(name = "licenses")
public void setLicenses(LicenseUrls licenses) {
this.licenses = licenses;
}
public ValidationUrls getValidations() {
return validations;
}
@XmlElement(name = "validators")
public void setValidations(ValidationUrls validations) {
this.validations = validations;
}
public Map<String, PrefillingConfig> getPrefillings() {
return prefillings;
}

48
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/QueryConfig.java
View File

@ -1,48 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
public class QueryConfig {
private String condition;
private String separator;
private String value;
private Integer ordinal;
public String getCondition() {
return condition;
}
@XmlElement(name = "condition")
public void setCondition(String condition) {
this.condition = condition;
}
public String getSeparator() {
return separator;
}
@XmlElement(name = "separator")
public void setSeparator(String separator) {
this.separator = separator;
}
public String getValue() {
return value;
}
@XmlElement(name = "value")
public void setValue(String value) {
this.value = value;
}
public Integer getOrdinal() {
return ordinal;
}
@XmlElement(name = "ordinal")
public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal;
}
}

128
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/UrlConfiguration.java
View File

@ -1,34 +1,23 @@
package eu.eudat.service.remotefetcher.config;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class UrlConfiguration {
public class UrlConfiguration implements SourceBaseConfiguration {
private String key;
private String label;
private Integer ordinal;
private String url;
private boolean dataInListing;
private DataUrlConfiguration data;
private String type;
private String paginationPath;
private String contentType;
private String funderQuery;
private String firstpage;
private String requestType = "GET";
private String requestBody = "";
private String filterType = "remote";
private AuthenticationConfiguration auth;
private List<QueryConfig> queries;
private ReferenceTypeSourceType type;
public String getKey() {
return key;
}
@XmlElement(name = "key")
public void setKey(String key) {
this.key = key;
}
@ -36,121 +25,24 @@ public class UrlConfiguration {
public String getLabel() {
return label;
}
@XmlElement(name = "label")
public void setLabel(String label) {
this.label = label;
}
public String getUrl() {
return url;
}
@XmlElement(name = "url")
public void setUrl(String url) {
this.url = url;
}
public boolean isDataInListing() {
return dataInListing;
}
@XmlElement(name = "dataInListing")
public void setDataInListing(boolean dataInListing) {
this.dataInListing = dataInListing;
}
public Integer getOrdinal() {
return ordinal;
}
@XmlElement(name = "ordinal")
public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal;
}
public DataUrlConfiguration getData() {
return data;
}
@XmlElement(name = "data")
public void setData(DataUrlConfiguration data) {
this.data = data;
}
public String getPaginationPath() {
return paginationPath;
}
@XmlElement(name = "paginationpath")
public void setPaginationPath(String paginationPath) {
this.paginationPath = paginationPath;
}
public String getType() {
public ReferenceTypeSourceType getType() {
return type;
}
@XmlElement(name = "type")
public void setType(String type) {
public void setType(ReferenceTypeSourceType type) {
this.type = type;
}
public String getContentType() {
return contentType;
}
@XmlElement(name = "contenttype")
public void setContentType(String contentType) {
this.contentType = contentType;
}
public String getFunderQuery() {
return funderQuery;
}
@XmlElement(name = "funderQuery")
public void setFunderQuery(String funderQuery) {
this.funderQuery = funderQuery;
}
public String getFirstpage() {
return firstpage;
}
@XmlElement(name = "firstPage")
public void setFirstpage(String firstpage) {
this.firstpage = firstpage;
}
public String getRequestType() {
return requestType;
}
@XmlElement(name = "request")
public void setRequestType(String requestType) {
this.requestType = requestType != null ? requestType : "GET";
}
public String getRequestBody() {
return requestBody;
}
@XmlElement(name = "requestBody")
public void setRequestBody(String requestBody) {
this.requestBody = requestBody != null ? requestBody : "";
}
public String getFilterType() {
return filterType;
}
@XmlElement(name = "filterType")
public void setFilterType(String filterType) {
this.filterType = filterType;
}
public List<QueryConfig> getQueries() {
return queries;
}
@XmlElementWrapper
@XmlElement(name = "query")
public void setQueries(List<QueryConfig> queries) {
this.queries = queries;
}
public AuthenticationConfiguration getAuth() {
return auth;
}
@XmlElement(name="authentication")
public void setAuth(AuthenticationConfiguration auth) {
this.auth = auth;
}
}

18
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/AuthenticationConfiguration.java Normal file
View File

@ -0,0 +1,18 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
public interface AuthenticationConfiguration {
Boolean getEnabled();
String getAuthUrl();
ReferenceTypeExternalApiHTTPMethodType getAuthMethod();
String getAuthTokenPath();
String getAuthRequestBody();
String getType();
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/DatasetUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class DatasetUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/FunderUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class FunderUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

31
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GeneralUrls.java
View File

@ -1,31 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import java.util.ArrayList;
import java.util.List;
public class GeneralUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public GeneralUrls() {
this.urls = new ArrayList<>();
}
@Override
public List<UrlConfiguration> getUrls() {
return urls;
}
@Override
public FetchStrategy getFetchMode() {
return fetchMode;
}
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

5
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java
View File

@ -1,12 +1,13 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import java.util.List;
public abstract class GenericUrls {
public abstract List<UrlConfiguration> getUrls();
public abstract List<SourceBaseConfiguration> getUrls();
public abstract FetchStrategy getFetchMode();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GrantUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class GrantUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/JournalUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class JournalUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/LicenseUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class LicenseUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/OrganisationUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class OrganisationUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

4
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java
View File

@ -20,8 +20,8 @@ public class PrefillingSearch extends GenericUrls {
}
@Override
public List<UrlConfiguration> getUrls() {
List<UrlConfiguration> urls = new ArrayList<>();
public List<SourceBaseConfiguration> getUrls() {
List<SourceBaseConfiguration> urls = new ArrayList<>();
urls.add(urlConfig);
return urls;
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ProjectUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ProjectUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PubRepositoryUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class PubRepositoryUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PublicationsUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class PublicationsUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

11
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/QueryConfig.java Normal file
View File

@ -0,0 +1,11 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface QueryConfig {
String getCondition();
String getSeparator();
String getValue();
Integer getOrdinal();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RegistryUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class RegistryUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RepositoryUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class RepositoryUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResearcherUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ResearcherUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

6
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultFieldsMappingConfiguration.java Normal file
View File

@ -0,0 +1,6 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface ResultFieldsMappingConfiguration {
String getResponsePath();
String getCode();
}

9
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultsConfiguration.java Normal file
View File

@ -0,0 +1,9 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
public interface ResultsConfiguration <T extends ResultFieldsMappingConfiguration> {
String getResultsArrayPath();
List<T> getFieldsMapping();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ServiceUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ServiceUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

10
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceBaseConfiguration.java Normal file
View File

@ -0,0 +1,10 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
public interface SourceBaseConfiguration {
String getKey();
String getLabel();
Integer getOrdinal();
ReferenceTypeSourceType getType();
}

29
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceExternalApiConfiguration.java Normal file
View File

@ -0,0 +1,29 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import java.util.List;
public interface SourceExternalApiConfiguration<RsConfig extends ResultsConfiguration<? extends ResultFieldsMappingConfiguration>, AuthConfig extends AuthenticationConfiguration, QConfig extends QueryConfig> extends SourceBaseConfiguration {
String getUrl();
RsConfig getResults();
String getPaginationPath();
String getContentType();
String getFirstPage();
ReferenceTypeExternalApiHTTPMethodType getHttpMethod();
String getRequestBody();
String getFilterType();
AuthConfig getAuth();
List<QConfig> getQueries();
}

7
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceStaticOptionConfiguration.java Normal file
View File

@ -0,0 +1,7 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
public interface SourceStaticOptionConfiguration <Option extends StaticOption> extends SourceBaseConfiguration {
List<Option> getOptions();
}

6
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/StaticOption.java Normal file
View File

@ -0,0 +1,6 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface StaticOption {
String getCode();
String getValue();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/TagUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
/**
* Created by ikalyvas on 7/9/2018.
*/
public class TagUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/TaxonomiesUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class TaxonomiesUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ValidationUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
public class ValidationUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

46
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/models/ExternalDataResult.java Normal file
View File

@ -0,0 +1,46 @@
package eu.eudat.service.remotefetcher.models;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class ExternalDataResult {
List<Map<String, String>> results;
List<Map<String, Object>> rawData;
public ExternalDataResult() {
this.results = new ArrayList<>();
}
public ExternalDataResult(List<Map<String, String>> results) {
this.results = results;
}
public List<Map<String, String>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
this.results = results;
}
public List<Map<String, Object>> getRawData() {
return rawData;
}
public void setRawData(List<Map<String, Object>> rawData) {
this.rawData = rawData;
}
public void addAll(ExternalDataResult other){
if (other == null) return;
if (other.getResults() != null){
if (this.getResults() == null) this.setResults(new ArrayList<>());
this.getResults().addAll(other.getResults());
}
if (other.getRawData() != null){
if (this.getRawData() == null) this.setRawData(new ArrayList<>());
this.getRawData().addAll(other.getRawData());
}
}
}

37
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/models/ExternalRefernceResult.java
View File

@ -1,37 +0,0 @@
package eu.eudat.service.remotefetcher.models;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ExternalRefernceResult {
List<Map<String, String>> results;
Map<String, Integer> pagination;
public ExternalRefernceResult() {
this.results = new ArrayList<>();
this.pagination = new HashMap<>();
}
public ExternalRefernceResult(List<Map<String, String>> results, Map<String, Integer> pagination) {
this.results = results;
this.pagination = pagination;
}
public List<Map<String, String>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
this.results = results;
}
public Map<String, Integer> getPagination() {
return pagination;
}
public void setPagination(Map<String, Integer> pagination) {
this.pagination = pagination;
}
}

5
dmp-backend/web/src/main/java/eu/eudat/controllers/v2/ReferenceController.java
View File

@ -11,7 +11,6 @@ import eu.eudat.model.censorship.ReferenceCensor;
import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.model.result.QueryResult;
import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.reference.ReferenceService;
@ -95,12 +94,12 @@ public class ReferenceController {
@PostMapping("search")
public List<Reference> searchReferenceWithDefinition(@RequestBody ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
public List<Reference> searchReferenceWithDefinition(@RequestBody ReferenceSearchLookup lookup) throws MyNotFoundException {
logger.debug("search with db definition {}", Reference.class.getSimpleName());
this.censorFactory.censor(ReferenceCensor.class).censor(lookup.getProject(), null);
List<Reference> references = this.referenceService.searchReferenceWithDefinition(lookup);
List<Reference> references = this.referenceService.searchReferenceData(lookup);
this.auditService.track(AuditableAction.Reference_Search, "lookup", lookup);