reference fetcher changes
This commit is contained in:
parent
87dd56ff2c
commit
957c1fad58
|
@ -1,9 +1,10 @@
|
|||
package eu.eudat.commons.types.referencetype;
|
||||
|
||||
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
|
||||
import eu.eudat.service.remotefetcher.config.entities.AuthenticationConfiguration;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
|
||||
public class AuthenticationConfigurationEntity {
|
||||
public class AuthenticationConfigurationEntity implements AuthenticationConfiguration {
|
||||
|
||||
private Boolean enabled;
|
||||
private String authUrl;
|
||||
|
|
|
@ -1,8 +1,9 @@
|
|||
package eu.eudat.commons.types.referencetype;
|
||||
|
||||
import eu.eudat.service.remotefetcher.config.entities.QueryConfig;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
|
||||
public class QueryConfigEntity {
|
||||
public class QueryConfigEntity implements QueryConfig {
|
||||
|
||||
private String condition;
|
||||
private String separator;
|
||||
|
|
|
@ -1,12 +1,13 @@
|
|||
package eu.eudat.commons.types.referencetype;
|
||||
|
||||
import eu.eudat.commons.enums.ReferenceTypeSourceType;
|
||||
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public abstract class ReferenceTypeSourceBaseConfigurationEntity {
|
||||
public abstract class ReferenceTypeSourceBaseConfigurationEntity implements SourceBaseConfiguration {
|
||||
|
||||
private String key;
|
||||
|
||||
|
|
|
@ -2,11 +2,12 @@ package eu.eudat.commons.types.referencetype;
|
|||
|
||||
|
||||
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
|
||||
import eu.eudat.service.remotefetcher.config.entities.SourceExternalApiConfiguration;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
|
||||
import java.util.List;
|
||||
public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity {
|
||||
public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceExternalApiConfiguration<ResultsConfigurationEntity, AuthenticationConfigurationEntity, QueryConfigEntity> {
|
||||
|
||||
private String url;
|
||||
private ResultsConfigurationEntity results;
|
||||
|
|
|
@ -1,11 +1,12 @@
|
|||
package eu.eudat.commons.types.referencetype;
|
||||
|
||||
import eu.eudat.service.remotefetcher.config.entities.SourceStaticOptionConfiguration;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity {
|
||||
public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceStaticOptionConfiguration<ReferenceTypeStaticOptionEntity> {
|
||||
|
||||
List<ReferenceTypeStaticOptionEntity> options;
|
||||
|
||||
|
|
|
@ -1,8 +1,9 @@
|
|||
package eu.eudat.commons.types.referencetype;
|
||||
|
||||
import eu.eudat.service.remotefetcher.config.entities.StaticOption;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
|
||||
public class ReferenceTypeStaticOptionEntity {
|
||||
public class ReferenceTypeStaticOptionEntity implements StaticOption {
|
||||
|
||||
private String code;
|
||||
|
||||
|
|
|
@ -1,8 +1,9 @@
|
|||
package eu.eudat.commons.types.referencetype;
|
||||
|
||||
import eu.eudat.service.remotefetcher.config.entities.ResultFieldsMappingConfiguration;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
|
||||
public class ResultFieldsMappingConfigurationEntity {
|
||||
public class ResultFieldsMappingConfigurationEntity implements ResultFieldsMappingConfiguration {
|
||||
private String code;
|
||||
private String responsePath;
|
||||
|
||||
|
|
|
@ -1,11 +1,12 @@
|
|||
package eu.eudat.commons.types.referencetype;
|
||||
|
||||
import eu.eudat.service.remotefetcher.config.entities.ResultsConfiguration;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public class ResultsConfigurationEntity {
|
||||
public class ResultsConfigurationEntity implements ResultsConfiguration<ResultFieldsMappingConfigurationEntity> {
|
||||
private String resultsArrayPath;
|
||||
private List<ResultFieldsMappingConfigurationEntity> fieldsMapping;
|
||||
|
||||
|
|
|
@ -1,58 +0,0 @@
|
|||
//package eu.eudat.model.builder.referencesearch;
|
||||
//
|
||||
//import eu.eudat.authorization.AuthorizationFlags;
|
||||
//import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
|
||||
//import eu.eudat.convention.ConventionService;
|
||||
//import eu.eudat.model.builder.BaseBuilder;
|
||||
//import eu.eudat.model.referencedefinition.Definition;
|
||||
//import gr.cite.tools.data.builder.BuilderFactory;
|
||||
//import gr.cite.tools.exception.MyApplicationException;
|
||||
//import gr.cite.tools.fieldset.FieldSet;
|
||||
//import gr.cite.tools.logging.DataLogEntry;
|
||||
//import gr.cite.tools.logging.LoggerService;
|
||||
//import org.slf4j.LoggerFactory;
|
||||
//import org.springframework.beans.factory.annotation.Autowired;
|
||||
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
|
||||
//import org.springframework.context.annotation.Scope;
|
||||
//import org.springframework.stereotype.Component;
|
||||
//
|
||||
//import java.util.*;
|
||||
//
|
||||
//@Component
|
||||
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
|
||||
//public class DefinitionSearchBuilder extends BaseBuilder<Definition, Map<String, String>> {
|
||||
//
|
||||
// private final BuilderFactory builderFactory;
|
||||
// private final ReferenceTypeProperties properties;
|
||||
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
|
||||
//
|
||||
// @Autowired
|
||||
// public DefinitionSearchBuilder(
|
||||
// ConventionService conventionService, BuilderFactory builderFactory, ReferenceTypeProperties properties) {
|
||||
// super(conventionService, new LoggerService(LoggerFactory.getLogger(DefinitionSearchBuilder.class)));
|
||||
// this.builderFactory = builderFactory;
|
||||
// this.properties = properties;
|
||||
// }
|
||||
//
|
||||
// public DefinitionSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
|
||||
// this.authorize = values;
|
||||
// return this;
|
||||
// }
|
||||
//
|
||||
// @Override
|
||||
// public List<Definition> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
|
||||
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
|
||||
// this.logger.trace(new DataLogEntry("requested fields", fields));
|
||||
// if (data == null)
|
||||
// return new ArrayList<>();
|
||||
//
|
||||
// List<Definition> models = new ArrayList<>();
|
||||
// //for (Map<String, String> d : data) {
|
||||
// Definition m = new Definition();
|
||||
// m.setFields(this.builderFactory.builder(FieldSearchBuilder.class).authorize(this.authorize).build(null, data));
|
||||
// models.add(m);
|
||||
// //}
|
||||
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
|
||||
// return models;
|
||||
// }
|
||||
//}
|
|
@ -1,88 +0,0 @@
|
|||
//package eu.eudat.model.builder.referencesearch;
|
||||
//
|
||||
//import eu.eudat.authorization.AuthorizationFlags;
|
||||
//import eu.eudat.commons.enums.ReferenceFieldDataType;
|
||||
//import eu.eudat.configurations.referencetype.ReferenceTypeField;
|
||||
//import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
|
||||
//import eu.eudat.convention.ConventionService;
|
||||
//import eu.eudat.model.builder.BaseBuilder;
|
||||
//import eu.eudat.model.referencedefinition.Field;
|
||||
//import gr.cite.tools.exception.MyApplicationException;
|
||||
//import gr.cite.tools.fieldset.FieldSet;
|
||||
//import gr.cite.tools.logging.DataLogEntry;
|
||||
//import gr.cite.tools.logging.LoggerService;
|
||||
//import org.slf4j.LoggerFactory;
|
||||
//import org.springframework.beans.factory.annotation.Autowired;
|
||||
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
|
||||
//import org.springframework.context.annotation.Scope;
|
||||
//import org.springframework.stereotype.Component;
|
||||
//
|
||||
//import java.util.*;
|
||||
//
|
||||
//@Component
|
||||
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
|
||||
//public class FieldSearchBuilder extends BaseBuilder<Field, Map<String, String>> {
|
||||
//
|
||||
// private final ReferenceTypeProperties properties;
|
||||
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
|
||||
//
|
||||
// @Autowired
|
||||
// public FieldSearchBuilder(
|
||||
// ConventionService conventionService, ReferenceTypeProperties properties) {
|
||||
// super(conventionService, new LoggerService(LoggerFactory.getLogger(FieldSearchBuilder.class)));
|
||||
// this.properties = properties;
|
||||
// }
|
||||
//
|
||||
// public FieldSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
|
||||
// this.authorize = values;
|
||||
// return this;
|
||||
// }
|
||||
//
|
||||
// @Override
|
||||
// public List<Field> build(FieldSet fields, List< Map<String, String>> data) throws MyApplicationException {
|
||||
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
|
||||
// this.logger.trace(new DataLogEntry("requested fields", fields));
|
||||
// if (data == null)
|
||||
// return new ArrayList<>();
|
||||
//
|
||||
// List<Field> models = new ArrayList<>();
|
||||
// for (Map<String, String> d : data) {
|
||||
//
|
||||
// ReferenceType referenceType = ReferenceType.valueOf(d.getOrDefault("referenceType", null));
|
||||
// List<ReferenceTypeField> typeFields = this.getPropertiesFields(referenceType);
|
||||
// if (typeFields.isEmpty()){
|
||||
// return new ArrayList<>();
|
||||
// }
|
||||
//
|
||||
// for (ReferenceTypeField typeField: typeFields){
|
||||
// Field m = new Field();
|
||||
// m.setCode(typeField.getCode());
|
||||
// m.setDataType(ReferenceFieldDataType.valueOf(typeField.getDataType()));
|
||||
// m.setValue(d.getOrDefault(typeField.getCode(), null));
|
||||
// models.add(m);
|
||||
// }
|
||||
// }
|
||||
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
|
||||
// return models;
|
||||
// }
|
||||
//
|
||||
// private List<ReferenceTypeField> getPropertiesFields(ReferenceType referenceType){
|
||||
// switch (referenceType) {
|
||||
// case Taxonomies: return properties.getTaxonomy().get("fields");
|
||||
// case Licenses: return properties.getLicence().get("fields");
|
||||
// case Publications: return properties.getPublication().get("fields");
|
||||
// case Journals: return properties.getJournal().get("fields");
|
||||
// case PubRepositories: return properties.getPubRepository().get("fields");
|
||||
// case DataRepositories: return properties.getDataRepository().get("fields");
|
||||
// case Registries: return properties.getRegistry().get("fields");
|
||||
// case Services: return properties.getService().get("fields");
|
||||
// case Organizations: return properties.getOrganisation().get("fields");
|
||||
// case Datasets: return properties.getDataset().get("fields");
|
||||
// case Funder: return properties.getFunder().get("fields");
|
||||
// case Grants: return properties.getGrant().get("fields");
|
||||
// case Project: return properties.getProject().get("fields");
|
||||
// case Researcher: return properties.getResearcher().get("fields");
|
||||
// default: return null;
|
||||
// }
|
||||
// }
|
||||
//}
|
|
@ -1,96 +0,0 @@
|
|||
//package eu.eudat.model.builder.referencesearch;
|
||||
//
|
||||
//import eu.eudat.authorization.AuthorizationFlags;
|
||||
//import eu.eudat.commons.enums.ReferenceSourceType;
|
||||
//import eu.eudat.convention.ConventionService;
|
||||
//import eu.eudat.model.Reference;
|
||||
//import eu.eudat.model.builder.BaseBuilder;
|
||||
//import eu.eudat.model.referencedefinition.Definition;
|
||||
//import eu.eudat.model.referencedefinition.Field;
|
||||
//import gr.cite.tools.data.builder.BuilderFactory;
|
||||
//import gr.cite.tools.exception.MyApplicationException;
|
||||
//import gr.cite.tools.fieldset.FieldSet;
|
||||
//import gr.cite.tools.logging.DataLogEntry;
|
||||
//import gr.cite.tools.logging.LoggerService;
|
||||
//import org.slf4j.LoggerFactory;
|
||||
//import org.springframework.beans.factory.annotation.Autowired;
|
||||
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
|
||||
//import org.springframework.context.annotation.Scope;
|
||||
//import org.springframework.stereotype.Component;
|
||||
//
|
||||
//import java.util.*;
|
||||
//
|
||||
//@Component
|
||||
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
|
||||
//public class ReferenceSearchBuilder extends BaseBuilder<Reference, Map<String, String>> {
|
||||
//
|
||||
// private final BuilderFactory builderFactory;
|
||||
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
|
||||
//
|
||||
// @Autowired
|
||||
// public ReferenceSearchBuilder(
|
||||
// ConventionService conventionService,
|
||||
// BuilderFactory builderFactory) {
|
||||
// super(conventionService, new LoggerService(LoggerFactory.getLogger(ReferenceSearchBuilder.class)));
|
||||
// this.builderFactory = builderFactory;
|
||||
// }
|
||||
//
|
||||
// public ReferenceSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
|
||||
// this.authorize = values;
|
||||
// return this;
|
||||
// }
|
||||
//
|
||||
// @Override
|
||||
// public List<Reference> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
|
||||
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
|
||||
// this.logger.trace(new DataLogEntry("requested fields", fields));
|
||||
// if (data == null)
|
||||
// return new ArrayList<>();
|
||||
//
|
||||
// List<Reference> models = new ArrayList<>();
|
||||
// for (Map<String, String> d : data) {
|
||||
// Reference m = new Reference();
|
||||
// if (d.containsKey("id")) m.setId(UUID.fromString(d.getOrDefault("id", null)));
|
||||
// m.setLabel(d.getOrDefault("name", null));
|
||||
// m.setSourceType(ReferenceSourceType.External);
|
||||
// String type = d.getOrDefault("referenceType", null);
|
||||
// if (type != null) m.setType(ReferenceType.valueOf(type));
|
||||
//
|
||||
// Definition definition = this.builderFactory.builder(DefinitionSearchBuilder.class).authorize(this.authorize).build(null, d);
|
||||
// m.setDefinition(definition);
|
||||
//
|
||||
// switch (m.getType()){
|
||||
// case Researcher:
|
||||
// case Organizations:
|
||||
// case Funder:
|
||||
// case Grants:
|
||||
// case DataRepositories:
|
||||
// case PubRepositories:
|
||||
// case Journals:
|
||||
// case Datasets:
|
||||
// case Registries:
|
||||
// case Services:{
|
||||
// for (Field field : definition.getFields()) {
|
||||
// if (field.getCode().equals("pid") && field.getValue() != null) {
|
||||
// if (d.containsKey("key")) {
|
||||
// m.setReference(d.get("key") + ":" + field.getValue());
|
||||
// }
|
||||
// }
|
||||
// if (field.getCode().equals("tag") && field.getValue() != null){
|
||||
// m.setSource(field.getValue());
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// m.setDescription(d.getOrDefault("description", null));
|
||||
// m.setAbbreviation(d.getOrDefault("abbreviation", null));
|
||||
//
|
||||
// models.add(m);
|
||||
//
|
||||
// }
|
||||
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
|
||||
// return models;
|
||||
// }
|
||||
//
|
||||
//}
|
|
@ -20,6 +20,7 @@ import eu.eudat.model.descriptionproperties.PropertyDefinitionFieldSetItem;
|
|||
import eu.eudat.model.persist.DescriptionProfilingRequest;
|
||||
import eu.eudat.model.persist.DescriptionProfilingWithDataRequest;
|
||||
import eu.eudat.service.remotefetcher.ExternalUrlConfigProvider;
|
||||
import eu.eudat.service.remotefetcher.RemoteFetcherService;
|
||||
import eu.eudat.service.remotefetcher.config.entities.*;
|
||||
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
||||
import gr.cite.tools.data.builder.BuilderFactory;
|
||||
|
@ -66,6 +67,7 @@ public class PrefillingServiceImpl implements PrefillingService {
|
|||
private final ExternalUrlConfigProvider externalUrlConfigProvider;
|
||||
private final XmlHandlingService xmlHandlingService;
|
||||
private final ValidatorFactory validatorFactory;
|
||||
private final RemoteFetcherService remoteFetcherService;
|
||||
@Autowired
|
||||
public PrefillingServiceImpl(
|
||||
EntityManager entityManager,
|
||||
|
@ -75,8 +77,9 @@ public class PrefillingServiceImpl implements PrefillingService {
|
|||
MessageSource messageSource,
|
||||
JsonHandlingService jsonHandlingService,
|
||||
ExternalUrlConfigProvider externalUrlConfigProvider,
|
||||
XmlHandlingService xmlHandlingService,
|
||||
ValidatorFactory validatorFactory) {
|
||||
XmlHandlingService xmlHandlingService,
|
||||
ValidatorFactory validatorFactory,
|
||||
RemoteFetcherService remoteFetcherService) {
|
||||
this.entityManager = entityManager;
|
||||
this.builderFactory = builderFactory;
|
||||
this.queryFactory = queryFactory;
|
||||
|
@ -86,6 +89,7 @@ public class PrefillingServiceImpl implements PrefillingService {
|
|||
this.externalUrlConfigProvider = externalUrlConfigProvider;
|
||||
this.xmlHandlingService = xmlHandlingService;
|
||||
this.validatorFactory = validatorFactory;
|
||||
this.remoteFetcherService = remoteFetcherService;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -98,7 +102,7 @@ public class PrefillingServiceImpl implements PrefillingService {
|
|||
List<Map<String, String>> map;
|
||||
Map<String, PrefillingConfig> prefillingConfigs = this.externalUrlConfigProvider.getExternalUrls().getPrefillings();
|
||||
// for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {//TODO new reference logic
|
||||
// map = remoteFetcherService.getExternalGeneric(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());
|
||||
// map = remoteFetcherService.getExternalData(prefillingConfig.getPrefillingSearch().getUrls(), externalReferenceCriteria, prefillingConfig.getPrefillingSearch().getFetchMode());
|
||||
// prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList());
|
||||
// if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) {
|
||||
// List<Map<String, Object>> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());
|
||||
|
|
|
@ -2,10 +2,12 @@ package eu.eudat.service.reference;
|
|||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import eu.eudat.commons.exceptions.HugeResultSetException;
|
||||
import eu.eudat.data.ReferenceTypeEntity;
|
||||
import eu.eudat.model.Reference;
|
||||
import eu.eudat.model.persist.ReferencePersist;
|
||||
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
|
||||
import eu.eudat.query.lookup.ReferenceSearchLookup;
|
||||
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
||||
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
|
||||
import gr.cite.tools.exception.MyApplicationException;
|
||||
import gr.cite.tools.exception.MyForbiddenException;
|
||||
import gr.cite.tools.exception.MyNotFoundException;
|
||||
|
@ -24,5 +26,5 @@ public interface ReferenceService {
|
|||
|
||||
void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException;
|
||||
|
||||
List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException;
|
||||
List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException;
|
||||
}
|
||||
|
|
|
@ -1,18 +1,12 @@
|
|||
package eu.eudat.service.reference;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.jayway.jsonpath.DocumentContext;
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
import eu.eudat.authorization.AuthorizationFlags;
|
||||
import eu.eudat.authorization.Permission;
|
||||
import eu.eudat.commons.XmlHandlingService;
|
||||
import eu.eudat.commons.enums.IsActive;
|
||||
import eu.eudat.commons.enums.ReferenceFieldDataType;
|
||||
import eu.eudat.commons.enums.ReferenceSourceType;
|
||||
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
|
||||
import eu.eudat.commons.enums.ReferenceTypeSourceType;
|
||||
import eu.eudat.commons.exceptions.HugeResultSetException;
|
||||
import eu.eudat.commons.types.reference.DefinitionEntity;
|
||||
import eu.eudat.commons.types.reference.FieldEntity;
|
||||
import eu.eudat.commons.types.referencetype.*;
|
||||
|
@ -20,17 +14,18 @@ import eu.eudat.convention.ConventionService;
|
|||
import eu.eudat.data.ReferenceEntity;
|
||||
import eu.eudat.data.ReferenceTypeEntity;
|
||||
import eu.eudat.model.Reference;
|
||||
import eu.eudat.model.ReferenceType;
|
||||
import eu.eudat.model.builder.ReferenceBuilder;
|
||||
import eu.eudat.model.deleter.ReferenceDeleter;
|
||||
import eu.eudat.model.persist.ReferencePersist;
|
||||
import eu.eudat.model.persist.referencedefinition.DefinitionPersist;
|
||||
import eu.eudat.model.persist.referencedefinition.FieldPersist;
|
||||
import eu.eudat.query.ReferenceQuery;
|
||||
import eu.eudat.query.ReferenceTypeQuery;
|
||||
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
|
||||
import eu.eudat.query.lookup.ReferenceSearchLookup;
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
|
||||
import eu.eudat.service.remotefetcher.RemoteFetcherService;
|
||||
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
|
||||
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
||||
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
|
||||
import gr.cite.commons.web.authz.service.AuthorizationService;
|
||||
import gr.cite.tools.data.builder.BuilderFactory;
|
||||
import gr.cite.tools.data.deleter.DeleterFactory;
|
||||
|
@ -45,28 +40,17 @@ import gr.cite.tools.logging.LoggerService;
|
|||
import gr.cite.tools.logging.MapLogEntry;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.xml.bind.JAXBException;
|
||||
import net.minidev.json.JSONArray;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.MessageSource;
|
||||
import org.springframework.context.i18n.LocaleContextHolder;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
|
||||
import org.springframework.http.codec.json.Jackson2JsonDecoder;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import reactor.netty.http.client.HttpClient;
|
||||
|
||||
import javax.management.InvalidApplicationException;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import javax.xml.transform.TransformerException;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class ReferenceServiceImpl implements ReferenceService {
|
||||
|
@ -81,17 +65,16 @@ public class ReferenceServiceImpl implements ReferenceService {
|
|||
private final QueryFactory queryFactory;
|
||||
private final XmlHandlingService xmlHandlingService;
|
||||
|
||||
private final WebClient client;
|
||||
|
||||
public final RemoteFetcherService remoteFetcherService;
|
||||
public ReferenceServiceImpl(
|
||||
EntityManager entityManager,
|
||||
AuthorizationService authorizationService,
|
||||
DeleterFactory deleterFactory,
|
||||
BuilderFactory builderFactory,
|
||||
ConventionService conventionService,
|
||||
MessageSource messageSource,
|
||||
QueryFactory queryFactory,
|
||||
XmlHandlingService xmlHandlingService) {
|
||||
EntityManager entityManager,
|
||||
AuthorizationService authorizationService,
|
||||
DeleterFactory deleterFactory,
|
||||
BuilderFactory builderFactory,
|
||||
ConventionService conventionService,
|
||||
MessageSource messageSource,
|
||||
QueryFactory queryFactory,
|
||||
XmlHandlingService xmlHandlingService, RemoteFetcherService remoteFetcherService) {
|
||||
this.entityManager = entityManager;
|
||||
this.authorizationService = authorizationService;
|
||||
this.deleterFactory = deleterFactory;
|
||||
|
@ -100,11 +83,7 @@ public class ReferenceServiceImpl implements ReferenceService {
|
|||
this.messageSource = messageSource;
|
||||
this.queryFactory = queryFactory;
|
||||
this.xmlHandlingService = xmlHandlingService;
|
||||
this.client = WebClient.builder().codecs(clientCodecConfigurer -> {
|
||||
clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
|
||||
clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
|
||||
}
|
||||
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
|
||||
this.remoteFetcherService = remoteFetcherService;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
@ -178,380 +157,340 @@ public class ReferenceServiceImpl implements ReferenceService {
|
|||
this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id));
|
||||
}
|
||||
|
||||
// public List<FetcherReference> searchReference(ReferenceType externalType, String query, String type) throws HugeResultSet, MyNotFoundException, InvalidApplicationException {
|
||||
// ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query);
|
||||
//
|
||||
// List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(externalType, externalUrlCriteria, type);
|
||||
//
|
||||
// List<FetcherReference> list = this.fetchFromDb(externalType, query, type, remoteRepos);
|
||||
//
|
||||
// list.addAll(remoteRepos.stream().map(FetcherReference::fromRemoteModel).toList());
|
||||
// list = list.stream().filter(x -> x.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
|
||||
// list.sort(Comparator.comparing(FetcherReference::getName));
|
||||
// return list;
|
||||
// }
|
||||
@Override
|
||||
public List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException {
|
||||
int initialOffset = 0;
|
||||
if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
|
||||
initialOffset = lookup.getPage().getOffset();
|
||||
lookup.getPage().setOffset(0);
|
||||
}
|
||||
|
||||
// @Override
|
||||
// public List<Reference> searchReference(ReferenceSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
|
||||
// int initialOffset = 0;
|
||||
// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
|
||||
// initialOffset = lookup.getPage().getOffset();
|
||||
// lookup.getPage().setOffset(0);
|
||||
// }
|
||||
//
|
||||
// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
|
||||
//
|
||||
// List<Map<String, String>> remoteRepos = remoteFetcherService.getReferences(lookup.getType(), externalReferenceCriteria, lookup.getKey());
|
||||
//
|
||||
// List<Reference> externalModels = this.builderFactory.builder(ReferenceSearchBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), remoteRepos);
|
||||
// List<Reference> models = this.fetchFromDb(lookup);
|
||||
//
|
||||
// models.addAll(externalModels);
|
||||
//
|
||||
// if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).collect(Collectors.toList()); }
|
||||
// models.sort(Comparator.comparing(Reference::getLabel));
|
||||
//
|
||||
// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
|
||||
// models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList();
|
||||
// }
|
||||
//
|
||||
// return models;
|
||||
// }
|
||||
ReferenceTypeEntity data = this.entityManager.find(ReferenceTypeEntity.class, lookup.getTypeId());
|
||||
if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{lookup.getTypeId(), ReferenceType.class.getSimpleName()}, LocaleContextHolder.getLocale()));
|
||||
|
||||
ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
|
||||
|
||||
private List<Reference> fetchFromDb(ReferenceSearchLookup lookup){
|
||||
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal);
|
||||
ExternalDataResult remoteRepos = this.getReferenceData(data, externalReferenceCriteria, lookup.getKey());
|
||||
|
||||
List<Reference> externalModels = new ArrayList<>();
|
||||
if (remoteRepos != null && this.conventionService.isListNullOrEmpty(remoteRepos.getResults())){
|
||||
List<ReferenceEntity> referenceEntities = new ArrayList<>();
|
||||
for (Map<String, String> result : remoteRepos.getResults()){
|
||||
if (result == null || result.isEmpty()) continue;;
|
||||
ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data, remoteRepos);
|
||||
referenceEntities.add(referenceEntity);
|
||||
}
|
||||
externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities);
|
||||
}
|
||||
|
||||
List<Reference> models = this.fetchReferenceFromDb(lookup);
|
||||
|
||||
for (Reference externalReference : externalModels){
|
||||
if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null &&x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference);
|
||||
}
|
||||
|
||||
if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).toList(); }
|
||||
models.sort(Comparator.comparing(Reference::getLabel));
|
||||
|
||||
if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
|
||||
models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList();
|
||||
}
|
||||
|
||||
return models;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) {
|
||||
ReferenceEntity referenceEntity = new ReferenceEntity(); //TODO new reference logic
|
||||
referenceEntity.setTypeId(data.getId());
|
||||
referenceEntity.setIsActive(IsActive.Active);
|
||||
referenceEntity.setReference(result.getOrDefault("key", null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("pid", null));
|
||||
referenceEntity.setSource(result.getOrDefault("tag", null));
|
||||
referenceEntity.setAbbreviation(result.getOrDefault("abbreviation", null));
|
||||
referenceEntity.setDescription(result.getOrDefault("description", null));
|
||||
referenceEntity.setLabel(result.getOrDefault("name", null));
|
||||
referenceEntity.setSourceType(ReferenceSourceType.External);
|
||||
DefinitionEntity definitionEntity = new DefinitionEntity();
|
||||
definitionEntity.setFields(new ArrayList<>());
|
||||
for (Map.Entry<String, String> resultValue : result.entrySet()){
|
||||
FieldEntity fieldEntity = new FieldEntity();
|
||||
fieldEntity.setCode(resultValue.getKey());
|
||||
fieldEntity.setValue(resultValue.getValue());
|
||||
fieldEntity.setDataType(ReferenceFieldDataType.Text);
|
||||
definitionEntity.getFields().add(fieldEntity);
|
||||
}
|
||||
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
|
||||
return referenceEntity;
|
||||
}
|
||||
|
||||
private List<Reference> fetchReferenceFromDb(ReferenceSearchLookup lookup){
|
||||
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId());
|
||||
List<ReferenceEntity> data = query.collectAs(lookup.getProject());
|
||||
return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data);
|
||||
}
|
||||
|
||||
// private List<FetcherReference> fetchFromDb(ReferenceType externalType, String query, String type, List<Map<String, String>> remoteRepos) throws InvalidApplicationException {
|
||||
// List<FetcherReference> list = new LinkedList<>();
|
||||
// switch (externalType) {
|
||||
// case DataRepositories:
|
||||
// case PubRepositories:
|
||||
// case Journals: {
|
||||
// DataRepositoryCriteria criteria = new DataRepositoryCriteria();
|
||||
// if (!query.isEmpty()) criteria.setLike(query);
|
||||
// criteria.setCreationUserId(this.userScope.getUserId());
|
||||
// if (type.equals("")) {
|
||||
// List<DataRepository> dataRepositoryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao().getWithCriteria(criteria)).toList();
|
||||
// list = dataRepositoryList.stream().map(item -> new FetcherReference().fromDataRepository(item)).collect(Collectors.toList());
|
||||
// }
|
||||
// }
|
||||
// case Registries: {
|
||||
// RegistryCriteria criteria = new RegistryCriteria();
|
||||
// if (!query.isEmpty()) criteria.setLike(query);
|
||||
// criteria.setCreationUserId(this.userScope.getUserId());
|
||||
// if (type.equals("")) {
|
||||
// List<Registry> registryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao().getWithCriteria(criteria)).toList();
|
||||
// list = registryList.stream().map(item -> new FetcherReference().fromRegistry(item)).collect(Collectors.toList());
|
||||
// }
|
||||
// }
|
||||
// case Services:
|
||||
// {
|
||||
// ServiceCriteria criteria = new ServiceCriteria();
|
||||
//
|
||||
// if (!query.isEmpty()) criteria.setLike(query);
|
||||
// criteria.setCreationUserId(this.userScope.getUserId());
|
||||
//
|
||||
// if (type.equals("")) {
|
||||
// List<Service> serviceList = (this.apiContext.getOperationsContext().getDatabaseRepository().getServiceDao().getWithCriteria(criteria)).toList();
|
||||
// list = serviceList.stream().map(item -> new FetcherReference().fromService(item)).collect(Collectors.toList());
|
||||
// }
|
||||
// }
|
||||
// case Datasets:{
|
||||
//
|
||||
// ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build();
|
||||
//
|
||||
// criteria.setCreationUserId(this.userScope.getUserId());
|
||||
// QueryableList<ExternalDataset> items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria);
|
||||
//
|
||||
// list = items.select(item -> new FetcherReference().fromDataset(item));
|
||||
// }
|
||||
// case Taxonomies:
|
||||
// case Publications:
|
||||
// case Licenses:
|
||||
// break;
|
||||
// }
|
||||
//
|
||||
// return list;
|
||||
// }
|
||||
|
||||
@Override
|
||||
public List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
|
||||
|
||||
ReferenceTypeQuery query = this.queryFactory.query(ReferenceTypeQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(lookup.getReferenceTypeId());
|
||||
List<ReferenceTypeEntity> datas = query.collectAs(lookup.getProject());
|
||||
if (datas.size() != 1 ){
|
||||
return null;
|
||||
private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) {
|
||||
ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition());
|
||||
if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult();
|
||||
|
||||
ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).toList(), externalReferenceCriteria, key, null);
|
||||
for (Map<String, String> result: results.getResults()) {
|
||||
result.put("referenceType", referenceType.getName());
|
||||
}
|
||||
ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, datas.get(0).getDefinition());
|
||||
List<Map<String, String>> remoteRepos = this.getAll(definition.getSources(), lookup);
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
|
||||
List<Map<String, String>> results = new LinkedList<>();
|
||||
|
||||
if (sources == null || sources.isEmpty()) {
|
||||
return results;
|
||||
}
|
||||
|
||||
sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
|
||||
|
||||
List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
|
||||
apiSources.forEach(source -> {
|
||||
try {
|
||||
String auth = null;
|
||||
if (source.getAuth()!= null) {
|
||||
//auth = this.getAuthentication(source.getAuth());
|
||||
}
|
||||
results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getLocalizedMessage(), e);
|
||||
}
|
||||
});
|
||||
List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
|
||||
staticSources.forEach(source -> {
|
||||
Map<String, String> map = new HashMap<>();
|
||||
source.getOptions().forEach(option -> {
|
||||
map.put(option.getCode(), option.getValue());
|
||||
map.put("tag", source.getLabel());
|
||||
map.put("key", source.getKey());
|
||||
});
|
||||
results.add(map);
|
||||
});
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
|
||||
HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
|
||||
Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
|
||||
.exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
|
||||
})).block();
|
||||
// private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
|
||||
// List<Map<String, String>> results = new LinkedList<>();
|
||||
//
|
||||
// if (sources == null || sources.isEmpty()) {
|
||||
// return results;
|
||||
// }
|
||||
//
|
||||
// sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
|
||||
//
|
||||
// List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
|
||||
// apiSources.forEach(source -> {
|
||||
// try {
|
||||
// String auth = null;
|
||||
// if (source.getAuth()!= null) {
|
||||
// //auth = this.getAuthentication(source.getAuth());
|
||||
// }
|
||||
// results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
|
||||
// } catch (Exception e) {
|
||||
// logger.error(e.getLocalizedMessage(), e);
|
||||
// }
|
||||
// });
|
||||
// List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
|
||||
// staticSources.forEach(source -> {
|
||||
// Map<String, String> map = new HashMap<>();
|
||||
// source.getOptions().forEach(option -> {
|
||||
// map.put(option.getCode(), option.getValue());
|
||||
// map.put("tag", source.getLabel());
|
||||
// map.put("key", source.getKey());
|
||||
// });
|
||||
// results.add(map);
|
||||
// });
|
||||
//
|
||||
// return results;
|
||||
// }
|
||||
|
||||
// private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
|
||||
// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
|
||||
// Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
|
||||
// .contentType(MediaType.APPLICATION_JSON)
|
||||
// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
|
||||
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
|
||||
// })).block();
|
||||
//
|
||||
//
|
||||
//
|
||||
// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
|
||||
// }
|
||||
|
||||
// private String parseBodyString(String bodyString) {
|
||||
// String finalBodyString = bodyString;
|
||||
// if (bodyString.contains("{env:")) {
|
||||
// int index = bodyString.indexOf("{env: ");
|
||||
// while (index >= 0) {
|
||||
// int endIndex = bodyString.indexOf("}", index + 6);
|
||||
// String envName = bodyString.substring(index + 6, endIndex);
|
||||
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
|
||||
// index = bodyString.indexOf("{env: ", index + 6);
|
||||
// }
|
||||
// }
|
||||
// return finalBodyString;
|
||||
// }
|
||||
|
||||
// private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
|
||||
// Set<Integer> pages = new HashSet<>();
|
||||
//
|
||||
// String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
|
||||
// String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
|
||||
//
|
||||
// ExternalDataResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
|
||||
// if(externalRefernceResult != null) {
|
||||
// if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
|
||||
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
|
||||
// .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
|
||||
// .collect(Collectors.toList()));
|
||||
// }
|
||||
// if (fetchStrategy == FetchStrategy.FIRST)
|
||||
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
|
||||
//
|
||||
// //Long maxResults = configLoader.getExternalUrls().getMaxresults();
|
||||
// Long maxResults = Long.valueOf(1000);
|
||||
// if (externalRefernceResult.getResults().size() > maxResults)
|
||||
// throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getResults().size() + " results... Please submit a more detailed search query");
|
||||
//
|
||||
// Optional<ExternalDataResult> optionalResults = pages.parallelStream()
|
||||
// .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
|
||||
// .filter(Objects::nonNull)
|
||||
// .reduce((result1, result2) -> {
|
||||
// result1.getResults().addAll(result2.getResults());
|
||||
// return result1;
|
||||
// });
|
||||
// ExternalDataResult remainingExternalDataResult = optionalResults.orElseGet(ExternalDataResult::new);
|
||||
// remainingExternalDataResult.getResults().addAll(externalRefernceResult.getResults());
|
||||
//
|
||||
// return remainingExternalDataResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
|
||||
// }
|
||||
// else {
|
||||
// return new LinkedList<>();
|
||||
// }
|
||||
// }
|
||||
|
||||
// private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
|
||||
// String completedPath = urlPath;
|
||||
//
|
||||
// if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
|
||||
// if (lookup.getLike() != null) {
|
||||
// completedPath = completedPath.replace("{like}", lookup.getLike());
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{like}", "*");
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (urlPath.contains("{like}")){
|
||||
// if (lookup.getLike() != null) {
|
||||
// completedPath = completedPath.replace("{like}", lookup.getLike());
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{like}", "");
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (urlPath.contains("{page}")) {
|
||||
// if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
|
||||
// completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
|
||||
// } else if (firstPage != null) {
|
||||
// completedPath = completedPath.replace("{page}", firstPage);
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{page}", "1");
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// if (urlPath.contains("{pageSize}")){
|
||||
// if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
|
||||
// completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{pageSize}", "100");
|
||||
// }
|
||||
// }
|
||||
//
|
||||
//
|
||||
// return completedPath;
|
||||
// }
|
||||
|
||||
// protected ExternalDataResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
|
||||
//
|
||||
// try {
|
||||
// ResponseEntity<String> response;
|
||||
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
|
||||
//
|
||||
// response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
|
||||
// if (contentType != null && !contentType.isEmpty()) {
|
||||
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
|
||||
// httpHeaders.setContentType(MediaType.valueOf(contentType));
|
||||
// }
|
||||
// if (auth != null) {
|
||||
// httpHeaders.set("Authorization", auth);
|
||||
// }
|
||||
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
|
||||
// if (response.getStatusCode() == HttpStatus.OK) { // success
|
||||
// ExternalDataResult externalDataResult = new ExternalDataResult();
|
||||
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
|
||||
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
|
||||
// externalDataResult = this.getFromJson(jsonContext, resultsEntity);
|
||||
// }
|
||||
//
|
||||
// return externalDataResult;
|
||||
// }
|
||||
// } catch (Exception exception) {
|
||||
// logger.error(exception.getMessage(), exception);
|
||||
// }
|
||||
//
|
||||
// return null;
|
||||
// }
|
||||
|
||||
|
||||
|
||||
return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
|
||||
}
|
||||
|
||||
private String parseBodyString(String bodyString) {
|
||||
String finalBodyString = bodyString;
|
||||
if (bodyString.contains("{env:")) {
|
||||
int index = bodyString.indexOf("{env: ");
|
||||
while (index >= 0) {
|
||||
int endIndex = bodyString.indexOf("}", index + 6);
|
||||
String envName = bodyString.substring(index + 6, endIndex);
|
||||
finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
|
||||
index = bodyString.indexOf("{env: ", index + 6);
|
||||
}
|
||||
}
|
||||
return finalBodyString;
|
||||
}
|
||||
|
||||
private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
|
||||
Set<Integer> pages = new HashSet<>();
|
||||
|
||||
String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
|
||||
String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
|
||||
|
||||
ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
|
||||
if(externalRefernceResult != null) {
|
||||
if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
|
||||
externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
|
||||
.filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
|
||||
.collect(Collectors.toList()));
|
||||
}
|
||||
if (fetchStrategy == FetchStrategy.FIRST)
|
||||
return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
|
||||
|
||||
if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set
|
||||
for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++)
|
||||
pages.add(i);
|
||||
|
||||
//Long maxResults = configLoader.getExternalUrls().getMaxresults();
|
||||
Long maxResults = Long.valueOf(1000);
|
||||
if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults))
|
||||
throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query");
|
||||
|
||||
Optional<ExternalRefernceResult> optionalResults = pages.parallelStream()
|
||||
.map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
|
||||
.filter(Objects::nonNull)
|
||||
.reduce((result1, result2) -> {
|
||||
result1.getResults().addAll(result2.getResults());
|
||||
return result1;
|
||||
});
|
||||
ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
|
||||
remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults());
|
||||
|
||||
return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
|
||||
}
|
||||
else {
|
||||
return new LinkedList<>();
|
||||
}
|
||||
}
|
||||
|
||||
private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
|
||||
String completedPath = urlPath;
|
||||
|
||||
if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
|
||||
if (lookup.getLike() != null) {
|
||||
completedPath = completedPath.replace("{like}", lookup.getLike());
|
||||
} else {
|
||||
completedPath = completedPath.replace("{like}", "*");
|
||||
}
|
||||
}
|
||||
|
||||
if (urlPath.contains("{like}")){
|
||||
if (lookup.getLike() != null) {
|
||||
completedPath = completedPath.replace("{like}", lookup.getLike());
|
||||
} else {
|
||||
completedPath = completedPath.replace("{like}", "");
|
||||
}
|
||||
}
|
||||
|
||||
if (urlPath.contains("{page}")) {
|
||||
if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
|
||||
completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
|
||||
} else if (firstPage != null) {
|
||||
completedPath = completedPath.replace("{page}", firstPage);
|
||||
} else {
|
||||
completedPath = completedPath.replace("{page}", "1");
|
||||
}
|
||||
}
|
||||
|
||||
if (urlPath.contains("{pageSize}")){
|
||||
if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
|
||||
completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
|
||||
} else {
|
||||
completedPath = completedPath.replace("{pageSize}", "100");
|
||||
}
|
||||
}
|
||||
// public static ExternalDataResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
|
||||
// return new ExternalDataResult(parseData(jsonContext, resultsEntity));
|
||||
// }
|
||||
|
||||
|
||||
return completedPath;
|
||||
}
|
||||
|
||||
protected ExternalRefernceResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
|
||||
|
||||
try {
|
||||
ResponseEntity<String> response;
|
||||
JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
|
||||
|
||||
response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
|
||||
if (contentType != null && !contentType.isEmpty()) {
|
||||
httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
|
||||
httpHeaders.setContentType(MediaType.valueOf(contentType));
|
||||
}
|
||||
if (auth != null) {
|
||||
httpHeaders.set("Authorization", auth);
|
||||
}
|
||||
}).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
|
||||
if (response.getStatusCode() == HttpStatus.OK) { // success
|
||||
ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult();
|
||||
if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
|
||||
DocumentContext jsonContext = JsonPath.parse(response.getBody());
|
||||
externalRefernceResult = this.getFromJson(jsonContext, resultsEntity);
|
||||
}
|
||||
|
||||
|
||||
if (externalRefernceResult.getPagination().size() == 0) {
|
||||
externalRefernceResult.getPagination().put("pages", 1);
|
||||
externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size());
|
||||
}
|
||||
return externalRefernceResult;
|
||||
}
|
||||
} catch (Exception exception) {
|
||||
logger.error(exception.getMessage(), exception);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
|
||||
return new ExternalRefernceResult(parseData(jsonContext, resultsEntity),
|
||||
new HashMap<>(1, 1));
|
||||
}
|
||||
|
||||
|
||||
private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
|
||||
List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
|
||||
List<Map<String, String>> parsedData = new ArrayList<>();
|
||||
|
||||
for (Map<String, String> stringObjectMap: rawData){
|
||||
Map<String, String> map = new HashMap<>();
|
||||
for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
|
||||
String pathValue = field.getResponsePath();
|
||||
if (!pathValue.contains(".")){
|
||||
if (stringObjectMap.containsKey(pathValue)) {
|
||||
//map.put(field.getCode(), stringObjectMap.get(pathValue));
|
||||
map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
|
||||
}
|
||||
}else {
|
||||
if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
|
||||
String value = null;
|
||||
Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
|
||||
if (fieldObj != null){
|
||||
if (fieldObj instanceof Map){
|
||||
Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
|
||||
if(o instanceof String){
|
||||
value = (String)o;
|
||||
}
|
||||
else if(o instanceof Integer){
|
||||
value = String.valueOf(o);
|
||||
}
|
||||
} else if (fieldObj instanceof List) {
|
||||
Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
|
||||
if(o instanceof String){
|
||||
value = (String)o;
|
||||
}
|
||||
else if(o instanceof Integer){
|
||||
value = String.valueOf(o);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (value != null){
|
||||
map.put(field.getCode(), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
parsedData.add(map);
|
||||
}
|
||||
return parsedData;
|
||||
}
|
||||
|
||||
|
||||
private static String normalizeValue(Object value) {
|
||||
if (value instanceof JSONArray) {
|
||||
JSONArray jarr = (JSONArray) value;
|
||||
if (jarr.get(0) instanceof String) {
|
||||
return jarr.get(0).toString();
|
||||
} else {
|
||||
for (Object o : jarr) {
|
||||
if ((o instanceof Map) && ((Map) o).containsKey("content")) {
|
||||
try {
|
||||
return ((Map<String, String>) o).get("content");
|
||||
}
|
||||
catch (ClassCastException e){
|
||||
if(((Map<?, ?>) o).get("content") instanceof Integer) {
|
||||
return String.valueOf(((Map<?, ?>) o).get("content"));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Map) {
|
||||
String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
|
||||
return ((Map<String, String>)value).get(key);
|
||||
}
|
||||
return value != null ? value.toString() : null;
|
||||
}
|
||||
// private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
|
||||
// List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
|
||||
// List<Map<String, String>> parsedData = new ArrayList<>();
|
||||
//
|
||||
// for (Map<String, String> stringObjectMap: rawData){
|
||||
// Map<String, String> map = new HashMap<>();
|
||||
// for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
|
||||
// String pathValue = field.getResponsePath();
|
||||
// if (!pathValue.contains(".")){
|
||||
// if (stringObjectMap.containsKey(pathValue)) {
|
||||
// //map.put(field.getCode(), stringObjectMap.get(pathValue));
|
||||
// map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
|
||||
// }
|
||||
// }else {
|
||||
// if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
|
||||
// String value = null;
|
||||
// Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
|
||||
// if (fieldObj != null){
|
||||
// if (fieldObj instanceof Map){
|
||||
// Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
|
||||
// if(o instanceof String){
|
||||
// value = (String)o;
|
||||
// }
|
||||
// else if(o instanceof Integer){
|
||||
// value = String.valueOf(o);
|
||||
// }
|
||||
// } else if (fieldObj instanceof List) {
|
||||
// Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
|
||||
// if(o instanceof String){
|
||||
// value = (String)o;
|
||||
// }
|
||||
// else if(o instanceof Integer){
|
||||
// value = String.valueOf(o);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if (value != null){
|
||||
// map.put(field.getCode(), value);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// parsedData.add(map);
|
||||
// }
|
||||
// return parsedData;
|
||||
// }
|
||||
//
|
||||
//
|
||||
// private static String normalizeValue(Object value) {
|
||||
// if (value instanceof JSONArray) {
|
||||
// JSONArray jarr = (JSONArray) value;
|
||||
// if (jarr.get(0) instanceof String) {
|
||||
// return jarr.get(0).toString();
|
||||
// } else {
|
||||
// for (Object o : jarr) {
|
||||
// if ((o instanceof Map) && ((Map) o).containsKey("content")) {
|
||||
// try {
|
||||
// return ((Map<String, String>) o).get("content");
|
||||
// }
|
||||
// catch (ClassCastException e){
|
||||
// if(((Map<?, ?>) o).get("content") instanceof Integer) {
|
||||
// return String.valueOf(((Map<?, ?>) o).get("content"));
|
||||
// }
|
||||
// return null;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// } else if (value instanceof Map) {
|
||||
// String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
|
||||
// return ((Map<String, String>)value).get(key);
|
||||
// }
|
||||
// return value != null ? value.toString() : null;
|
||||
// }
|
||||
|
||||
}
|
||||
|
|
|
@ -18,7 +18,6 @@ import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl;
|
|||
import gr.cite.commons.web.authz.service.AuthorizationService;
|
||||
import gr.cite.tools.data.builder.BuilderFactory;
|
||||
import gr.cite.tools.data.deleter.DeleterFactory;
|
||||
import gr.cite.tools.data.query.QueryFactory;
|
||||
import gr.cite.tools.exception.MyApplicationException;
|
||||
import gr.cite.tools.exception.MyForbiddenException;
|
||||
import gr.cite.tools.exception.MyNotFoundException;
|
||||
|
|
|
@ -1,21 +1,14 @@
|
|||
//package eu.eudat.service.remotefetcher;
|
||||
//
|
||||
//import eu.eudat.commons.exceptions.HugeResultSetException;
|
||||
//import eu.eudat.service.remotefetcher.config.entities.GenericUrls;
|
||||
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
||||
//import gr.cite.tools.exception.MyNotFoundException;
|
||||
//
|
||||
//import java.util.List;
|
||||
//import java.util.Map;
|
||||
//
|
||||
//public interface RemoteFetcherService {
|
||||
// List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException;
|
||||
//
|
||||
// GenericUrls getExternalUrls(ReferenceType referenceType);
|
||||
//
|
||||
// Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException;
|
||||
//
|
||||
// List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
|
||||
//
|
||||
// List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
|
||||
//}
|
||||
package eu.eudat.service.remotefetcher;
|
||||
|
||||
import eu.eudat.data.ReferenceTypeEntity;
|
||||
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
|
||||
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface RemoteFetcherService {
|
||||
ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy);
|
||||
Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
|
||||
}
|
||||
|
|
|
@ -1,533 +1,388 @@
|
|||
//package eu.eudat.service.remotefetcher;
|
||||
//
|
||||
//import com.fasterxml.jackson.core.type.TypeReference;
|
||||
//import com.fasterxml.jackson.databind.JsonNode;
|
||||
//import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
//import com.jayway.jsonpath.DocumentContext;
|
||||
//import com.jayway.jsonpath.JsonPath;
|
||||
//import eu.eudat.commons.exceptions.HugeResultSetException;
|
||||
//import eu.eudat.service.remotefetcher.config.AuthenticationConfiguration;
|
||||
//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
|
||||
//import eu.eudat.service.remotefetcher.config.QueryConfig;
|
||||
//import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
//import eu.eudat.service.remotefetcher.config.entities.GenericUrls;
|
||||
//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
|
||||
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
||||
//import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
//import gr.cite.tools.exception.MyNotFoundException;
|
||||
//import jakarta.xml.bind.JAXBContext;
|
||||
//import jakarta.xml.bind.Unmarshaller;
|
||||
//import org.slf4j.Logger;
|
||||
//import org.slf4j.LoggerFactory;
|
||||
//import org.springframework.beans.factory.annotation.Autowired;
|
||||
//import org.springframework.core.ParameterizedTypeReference;
|
||||
//import org.springframework.http.*;
|
||||
//import org.springframework.http.client.reactive.ReactorClientHttpConnector;
|
||||
//import org.springframework.http.codec.json.Jackson2JsonDecoder;
|
||||
//import org.springframework.stereotype.Service;
|
||||
//import org.springframework.web.client.RestTemplate;
|
||||
//import org.springframework.web.reactive.function.client.WebClient;
|
||||
//import reactor.netty.http.client.HttpClient;
|
||||
//
|
||||
//import java.io.File;
|
||||
//import java.io.StringReader;
|
||||
//import java.lang.reflect.Method;
|
||||
//import java.nio.file.Paths;
|
||||
//import java.util.*;
|
||||
//import java.util.stream.Collectors;
|
||||
//
|
||||
//@Service
|
||||
//public class RemoteFetcherServiceImpl implements RemoteFetcherService {
|
||||
// private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
|
||||
//
|
||||
// private WebClient webClient;
|
||||
// private final ExternalUrlConfigProvider externalUrlConfigProvider;
|
||||
// @Autowired
|
||||
// public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider) {
|
||||
// this.externalUrlConfigProvider = externalUrlConfigProvider;
|
||||
// }
|
||||
//
|
||||
// private WebClient getWebClient(){
|
||||
// if (this.webClient == null) {
|
||||
// this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> {
|
||||
// clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
|
||||
// clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
|
||||
// }
|
||||
// ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
|
||||
// }
|
||||
// return webClient;
|
||||
// }
|
||||
//
|
||||
// @Override
|
||||
// public List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException {
|
||||
// FetchStrategy fetchStrategy = null;
|
||||
//
|
||||
//// GenericUrls exGenericUrls = this.getExternalUrls(referenceType);
|
||||
// List<UrlConfiguration> urlConfigs = key != null && !key.isEmpty() ? exGenericUrls.getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
||||
// : exGenericUrls.getUrls();
|
||||
//
|
||||
// List<Map<String, String>> results = getAll(urlConfigs, fetchStrategy, externalReferenceCriteria);
|
||||
// for (Map<String, String> result: results) {
|
||||
// result.put("referenceType", referenceType.name());
|
||||
// }
|
||||
// return results;
|
||||
// }
|
||||
//
|
||||
// @Override
|
||||
// public GenericUrls getExternalUrls(ReferenceType referenceType) {
|
||||
// return switch (referenceType) {
|
||||
// case Taxonomies -> this.externalUrlConfigProvider.getExternalUrls().getTaxonomies();
|
||||
// case Licenses -> this.externalUrlConfigProvider.getExternalUrls().getLicenses();
|
||||
// case Publications -> this.externalUrlConfigProvider.getExternalUrls().getPublications();
|
||||
// case Journals -> this.externalUrlConfigProvider.getExternalUrls().getJournals();
|
||||
// case PubRepositories -> this.externalUrlConfigProvider.getExternalUrls().getPubRepositories();
|
||||
// case DataRepositories -> this.externalUrlConfigProvider.getExternalUrls().getRepositories();
|
||||
// case Registries -> this.externalUrlConfigProvider.getExternalUrls().getRegistries();
|
||||
// case Services -> this.externalUrlConfigProvider.getExternalUrls().getServices();
|
||||
// case Grants -> this.externalUrlConfigProvider.getExternalUrls().getGrants();
|
||||
// case Organizations -> this.externalUrlConfigProvider.getExternalUrls().getOrganisations();
|
||||
// case Datasets -> this.externalUrlConfigProvider.getExternalUrls().getDatasets();
|
||||
// case Funder -> this.externalUrlConfigProvider.getExternalUrls().getFunders();
|
||||
// case Project -> this.externalUrlConfigProvider.getExternalUrls().getProjects();
|
||||
// case Researcher -> this.externalUrlConfigProvider.getExternalUrls().getResearchers();
|
||||
// default -> throw new IllegalArgumentException("Type not found" + referenceType);
|
||||
// };
|
||||
// }
|
||||
//
|
||||
// @Override
|
||||
// public Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException {
|
||||
// List<UrlConfiguration> urlConfigs =
|
||||
// key != null && !key.isEmpty() ? this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList())
|
||||
// : this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls();
|
||||
// FetchStrategy fetchStrategy = this.externalUrlConfigProvider.getExternalUrls().getValidations().getFetchMode();
|
||||
// List<Map<String, String>> data = this.getAll(urlConfigs, fetchStrategy, externalReferenceCriteria);
|
||||
// return data.size();
|
||||
// }
|
||||
//
|
||||
// @Override
|
||||
// public List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) {
|
||||
// List<UrlConfiguration> urlConfigurations = genericUrls.getUrls();
|
||||
// FetchStrategy fetchStrategy = genericUrls.getFetchMode();
|
||||
// return getAll(urlConfigurations, fetchStrategy, externalReferenceCriteria);
|
||||
// }
|
||||
//
|
||||
// @Override
|
||||
// public List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) {
|
||||
// List<UrlConfiguration> urlConfigurations = genericUrls.getUrls();
|
||||
// return getAllWithData(urlConfigurations, externalReferenceCriteria);
|
||||
// }
|
||||
//
|
||||
// private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
|
||||
//
|
||||
// List<Map<String, String>> results = new LinkedList<>();
|
||||
//
|
||||
// if (urlConfigs == null || urlConfigs.isEmpty()) {
|
||||
// return results;
|
||||
// }
|
||||
//// throw new MyNotFoundException("No Repository urls found in configuration");
|
||||
//
|
||||
// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
|
||||
// for (UrlConfiguration urlConfiguration : urlConfigs) {
|
||||
// applyFunderQuery(urlConfiguration, externalReferenceCriteria);
|
||||
// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
|
||||
// try {
|
||||
// String auth = null;
|
||||
// if (urlConfiguration.getAuth() != null) {
|
||||
// auth = this.getAuthentication(urlConfiguration.getAuth());
|
||||
// }
|
||||
// results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalReferenceCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType(), urlConfiguration.getQueries(), auth));
|
||||
// } catch (Exception e) {
|
||||
// logger.error(e.getLocalizedMessage(), e);
|
||||
// }
|
||||
// } else if (urlConfiguration.getType() != null && urlConfiguration.getType().equals("Internal")) {
|
||||
// results.addAll(getAllResultsFromMockUpJson(urlConfiguration.getUrl(), externalReferenceCriteria.getLike()));
|
||||
package eu.eudat.service.remotefetcher;
|
||||
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.jayway.jsonpath.DocumentContext;
|
||||
import com.jayway.jsonpath.JsonPath;
|
||||
import eu.eudat.commons.XmlHandlingService;
|
||||
import eu.eudat.commons.enums.ReferenceTypeSourceType;
|
||||
import eu.eudat.commons.exceptions.HugeResultSetException;
|
||||
import eu.eudat.convention.ConventionService;
|
||||
import eu.eudat.service.remotefetcher.config.entities.*;
|
||||
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
|
||||
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import gr.cite.tools.exception.MyApplicationException;
|
||||
import net.minidev.json.JSONArray;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.core.ParameterizedTypeReference;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
|
||||
import org.springframework.http.codec.json.Jackson2JsonDecoder;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.web.reactive.function.client.WebClient;
|
||||
import reactor.netty.http.client.HttpClient;
|
||||
|
||||
import java.io.File;
|
||||
import java.nio.file.Paths;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class RemoteFetcherServiceImpl implements RemoteFetcherService {
|
||||
private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
|
||||
|
||||
private WebClient webClient;
|
||||
private final ExternalUrlConfigProvider externalUrlConfigProvider;
|
||||
private final ConventionService conventionService;
|
||||
private final XmlHandlingService xmlHandlingService;
|
||||
@Autowired
|
||||
public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) {
|
||||
this.externalUrlConfigProvider = externalUrlConfigProvider;
|
||||
this.conventionService = conventionService;
|
||||
this.xmlHandlingService = xmlHandlingService;
|
||||
}
|
||||
|
||||
private WebClient getWebClient(){
|
||||
if (this.webClient == null) {
|
||||
this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> {
|
||||
clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
|
||||
clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
|
||||
}
|
||||
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
|
||||
}
|
||||
return webClient;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Override
|
||||
public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) {
|
||||
List<SourceBaseConfiguration> apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList();
|
||||
if (!this.conventionService.isNullOrEmpty(key)){
|
||||
apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList();
|
||||
}
|
||||
if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult();
|
||||
|
||||
apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal));
|
||||
|
||||
return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
|
||||
return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size();
|
||||
}
|
||||
|
||||
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
|
||||
|
||||
ExternalDataResult results = new ExternalDataResult();
|
||||
|
||||
if (this.conventionService.isListNullOrEmpty(sources)) return new ExternalDataResult();
|
||||
|
||||
for (SourceBaseConfiguration source : sources) {
|
||||
if (source.getType() == null || source.getType().equals(ReferenceTypeSourceType.API)) {
|
||||
try {
|
||||
SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource = (SourceExternalApiConfiguration)source;
|
||||
this.applyFunderToQuery(apiSource, externalReferenceCriteria);
|
||||
|
||||
String auth = null;
|
||||
if (apiSource.getAuth() != null) {
|
||||
auth = this.buildAuthentication(apiSource.getAuth());
|
||||
}
|
||||
results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth));
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getLocalizedMessage(), e);
|
||||
}
|
||||
} else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) {
|
||||
SourceStaticOptionConfiguration<StaticOption> staticSource = (SourceStaticOptionConfiguration)source;
|
||||
results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO:
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) {
|
||||
HttpMethod method;
|
||||
switch (authenticationConfiguration.getAuthMethod()) {
|
||||
case GET -> method = HttpMethod.GET;
|
||||
case POST -> method =HttpMethod.POST;
|
||||
default -> throw new MyApplicationException("unrecognized type " + authenticationConfiguration.getAuthMethod());
|
||||
}
|
||||
|
||||
Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
|
||||
.contentType(MediaType.APPLICATION_JSON)
|
||||
.bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
|
||||
.exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
|
||||
})).block();
|
||||
if (response == null) throw new MyApplicationException("Authentication " + authenticationConfiguration.getAuthUrl() + " failed");
|
||||
|
||||
return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null);
|
||||
}
|
||||
|
||||
|
||||
private void applyFunderToQuery(SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) {
|
||||
//TODO new reference logic
|
||||
// if (apiSource.getFunderQuery() != null) {
|
||||
// if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) {
|
||||
// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery()));
|
||||
// }
|
||||
// else {
|
||||
// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", ""));
|
||||
// }
|
||||
// }
|
||||
// /* for (UrlConfiguration urlConfig : urlConfigs) {
|
||||
// ifFunderQueryExist(urlConfig, externalUrlCriteria);
|
||||
// if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
|
||||
// results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType()));
|
||||
// } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
|
||||
// results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
|
||||
// }
|
||||
// }*/
|
||||
// return results;
|
||||
// }
|
||||
//
|
||||
// private String getAuthentication(AuthenticationConfiguration authenticationConfiguration) {
|
||||
// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod());
|
||||
// Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
|
||||
// .contentType(MediaType.APPLICATION_JSON)
|
||||
// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
|
||||
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
|
||||
// })).block();
|
||||
//
|
||||
//
|
||||
//
|
||||
// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
|
||||
// }
|
||||
//
|
||||
// private List<Map<String, Object>> getAllWithData(List<UrlConfiguration> urlConfigs, ExternalReferenceCriteria externalReferenceCriteria) {
|
||||
//
|
||||
// List<Map<String, Object>> results = new LinkedList<>();
|
||||
//
|
||||
// if (urlConfigs == null || urlConfigs.isEmpty()) {
|
||||
// return results;
|
||||
// }
|
||||
//
|
||||
// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
|
||||
// urlConfigs.forEach(urlConfiguration -> {
|
||||
// applyFunderQuery(urlConfiguration, externalReferenceCriteria);
|
||||
// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
|
||||
// try {
|
||||
// results.addAll(getAllResultsFromUrlWithData(urlConfiguration.getUrl(), urlConfiguration.getData(), externalReferenceCriteria, urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getQueries()));
|
||||
// } catch (Exception e) {
|
||||
// logger.error(e.getLocalizedMessage(), e);
|
||||
// }
|
||||
// }
|
||||
// });
|
||||
// return results;
|
||||
//
|
||||
// }
|
||||
//
|
||||
// private void applyFunderQuery(UrlConfiguration urlConfiguration, ExternalReferenceCriteria externalReferenceCriteria) {
|
||||
// if (urlConfiguration.getFunderQuery() != null) {
|
||||
// if (externalReferenceCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) {
|
||||
// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery()));
|
||||
// }
|
||||
// else {
|
||||
// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", ""));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// private String calculateQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
|
||||
// String finalQuery = "";
|
||||
// QueryConfig queryConfig = queryConfigs.stream().filter(queryConfigl -> externalReferenceCriteria.getLike().matches(queryConfigl.getCondition()))
|
||||
// .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
|
||||
// if (queryConfig != null) {
|
||||
// if (queryConfig.getSeparator() != null) {
|
||||
// String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
|
||||
// finalQuery = queryConfig.getValue();
|
||||
// for (int i = 0; i < likes.length; i++) {
|
||||
// finalQuery = finalQuery.replaceAll("\\{like" + (i+1) + "}", likes[i]);
|
||||
// }
|
||||
// } else {
|
||||
// finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike());
|
||||
// }
|
||||
//
|
||||
// }
|
||||
// return finalQuery;
|
||||
// }
|
||||
//
|
||||
// protected String replaceCriteriaOnUrl(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
|
||||
// String completedPath = path;
|
||||
// if (externalReferenceCriteria.getLike() != null) {
|
||||
// if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().equals("")) {
|
||||
// completedPath = completedPath.replaceAll("\\{like}", "*");
|
||||
// completedPath = completedPath.replaceAll("\\{query}", "*");
|
||||
// } else {
|
||||
// if (completedPath.contains("{query}")) {
|
||||
// completedPath = completedPath.replaceAll("\\{query}", this.calculateQuery(externalReferenceCriteria, queries));
|
||||
// } else {
|
||||
// completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike());
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{like}", "");
|
||||
// }
|
||||
// if (externalReferenceCriteria.getFunderId() != null) {
|
||||
// String funderPrefix = externalReferenceCriteria.getFunderId().split(":")[0];
|
||||
// String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", "");
|
||||
// if (funderId.toCharArray()[0] == ':') {
|
||||
// funderId = externalReferenceCriteria.getFunderId();
|
||||
// }
|
||||
// /*
|
||||
// try { funderId = URLEncoder.encode(funderId, "UTF-8"); } catch
|
||||
// (UnsupportedEncodingException e) { logger.error(e.getMessage(), e); }
|
||||
// */
|
||||
// completedPath = completedPath.replace("{funderId}", funderId);
|
||||
// }
|
||||
// else if(completedPath.contains("{funderId}")){
|
||||
// logger.warn("FunderId is null.");
|
||||
// completedPath = completedPath.replace("{funderId}", " ");
|
||||
// }
|
||||
// if (externalReferenceCriteria.getPage() != null) {
|
||||
// completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
|
||||
// } else {
|
||||
// if (firstPage != null) {
|
||||
// completedPath = completedPath.replace("{page}", firstPage);
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{page}", "1");
|
||||
// }
|
||||
// }
|
||||
// if (externalReferenceCriteria.getPageSize() != null) {
|
||||
// completedPath = completedPath.replace("{pageSize}", externalReferenceCriteria.getPageSize());
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{pageSize}", "60");
|
||||
// }
|
||||
// if (externalReferenceCriteria.getHost() != null) {
|
||||
// completedPath = completedPath.replace("{host}", externalReferenceCriteria.getHost());
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{host}", "");
|
||||
// }
|
||||
// if (externalReferenceCriteria.getPath() != null) {
|
||||
// completedPath = completedPath.replace("{path}", externalReferenceCriteria.getPath());
|
||||
// } else {
|
||||
// completedPath = completedPath.replace("{path}", "");
|
||||
// }
|
||||
// return completedPath;
|
||||
// }
|
||||
//
|
||||
// private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalReferenceCriteria externalReferenceCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType, List<QueryConfig> queries, String auth) throws Exception {
|
||||
// Set<Integer> pages = new HashSet<>();
|
||||
//
|
||||
// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries);
|
||||
// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries);
|
||||
//
|
||||
// ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth);
|
||||
// if(externalRefernceResult != null) {
|
||||
// if (filterType != null && filterType.equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
|
||||
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
|
||||
// .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
|
||||
// .collect(Collectors.toList()));
|
||||
// }
|
||||
// if (fetchStrategy == FetchStrategy.FIRST)
|
||||
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
|
||||
//
|
||||
// if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set
|
||||
// for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++)
|
||||
// pages.add(i);
|
||||
//
|
||||
// Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
|
||||
// if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults))
|
||||
// throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query");
|
||||
//
|
||||
// Optional<ExternalRefernceResult> optionalResults = pages.parallelStream()
|
||||
// .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth))
|
||||
// .filter(Objects::nonNull)
|
||||
// .reduce((result1, result2) -> {
|
||||
// result1.getResults().addAll(result2.getResults());
|
||||
// return result1;
|
||||
// });
|
||||
// ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
|
||||
// remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults());
|
||||
//
|
||||
// return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
|
||||
// }
|
||||
// else {
|
||||
// return new LinkedList<>();
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// private List<Map<String, Object>> getAllResultsFromUrlWithData(String path, final DataUrlConfiguration jsonDataPath, ExternalReferenceCriteria externalReferenceCriteria, String contentType, String firstPage, String requestBody, String requestType, List<QueryConfig> queries) {
|
||||
//
|
||||
// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries);
|
||||
// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries);
|
||||
//
|
||||
// try {
|
||||
// RestTemplate restTemplate = new RestTemplate();
|
||||
// HttpHeaders headers = new HttpHeaders();
|
||||
// HttpEntity<JsonNode> entity;
|
||||
// ResponseEntity<String> response;
|
||||
// if (contentType != null && !contentType.isEmpty()) {
|
||||
// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
|
||||
// headers.setContentType(MediaType.valueOf(contentType));
|
||||
// }
|
||||
// JsonNode jsonBody = new ObjectMapper().readTree(replacedBody);
|
||||
// entity = new HttpEntity<>(jsonBody, headers);
|
||||
//
|
||||
// response = restTemplate.exchange(replacedPath, HttpMethod.valueOf(requestType), entity, String.class);
|
||||
// if (response.getStatusCode() == HttpStatus.OK) {
|
||||
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
|
||||
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
|
||||
// return jsonContext.read(jsonDataPath.getPath());
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// catch (Exception exception) {
|
||||
// logger.error(exception.getMessage(), exception);
|
||||
// }
|
||||
//
|
||||
// return new LinkedList<>();
|
||||
// }
|
||||
//
|
||||
//
|
||||
// protected ExternalRefernceResult getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType, String auth) {
|
||||
//
|
||||
// try {
|
||||
// //RestTemplate restTemplate = new RestTemplate(new SimpleClientHttpRequestFactory());
|
||||
// //HttpHeaders headers = new HttpHeaders();
|
||||
// //HttpEntity<JsonNode> entity;
|
||||
// ResponseEntity<String> response;
|
||||
// /*
|
||||
// * URL url = new URL(urlString.replaceAll(" ", "%20"));
|
||||
// *
|
||||
// * HttpURLConnection con = (HttpURLConnection) url.openConnection();
|
||||
// * con.setRequestMethod("GET");
|
||||
// */
|
||||
// /* if (contentType != null && !contentType.isEmpty()) {
|
||||
// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
|
||||
// headers.setContentType(MediaType.valueOf(contentType));
|
||||
// }
|
||||
// if (auth != null) {
|
||||
// headers.set("Authorization", auth);
|
||||
// }*/
|
||||
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
|
||||
//// entity = new HttpEntity<>(jsonBody, headers);
|
||||
//
|
||||
//
|
||||
// response = this.getWebClient().method(HttpMethod.valueOf(requestType)).uri(urlString).headers(httpHeaders -> {
|
||||
// if (contentType != null && !contentType.isEmpty()) {
|
||||
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
|
||||
// httpHeaders.setContentType(MediaType.valueOf(contentType));
|
||||
// }
|
||||
// if (auth != null) {
|
||||
// httpHeaders.set("Authorization", auth);
|
||||
// }
|
||||
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
|
||||
// //response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class);
|
||||
// if (response.getStatusCode() == HttpStatus.OK) { // success
|
||||
// //do here all the parsing
|
||||
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult();
|
||||
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
|
||||
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
|
||||
//
|
||||
// if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
|
||||
// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType, auth);
|
||||
// } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
|
||||
// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
|
||||
// } else {
|
||||
// externalRefernceResult = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath);
|
||||
// }
|
||||
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
|
||||
// .collect(Collectors.toList()));
|
||||
// }
|
||||
// else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) {
|
||||
// Class<?> aClass = Class.forName(jsonDataPath.getParseClass());
|
||||
// JAXBContext jaxbContext = JAXBContext.newInstance(aClass);
|
||||
// Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
|
||||
// StringReader stringReader = new StringReader(response.getBody());
|
||||
// Object data = unmarshaller.unmarshal(stringReader);
|
||||
// Method reader = null;
|
||||
// if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) {
|
||||
// String camelCaseGetter = jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty() ? "get" + jsonDataPath.getParseField().substring(0, 1).toUpperCase() + jsonDataPath.getParseField().substring(1) : "";
|
||||
// reader = aClass.getMethod(camelCaseGetter);
|
||||
// }
|
||||
// ObjectMapper objectMapper = new ObjectMapper();
|
||||
// List<Map<String, String>> values = new ArrayList<>();
|
||||
// int max = 1;
|
||||
// if (reader != null) {
|
||||
// Object invokedField = reader.invoke(data);
|
||||
// if (invokedField instanceof Collection) {
|
||||
// max = ((Collection) invokedField).size();
|
||||
// }
|
||||
// }
|
||||
// for (int i = 0; i< max; i++) {
|
||||
// Object value;
|
||||
// if (reader != null) {
|
||||
// Object invokedField = reader.invoke(data);
|
||||
// if (invokedField instanceof Collection) {
|
||||
// value = ((Collection) invokedField).toArray()[i];
|
||||
// } else {
|
||||
// value = invokedField;
|
||||
// }
|
||||
// } else {
|
||||
// value = data;
|
||||
// }
|
||||
// Map<String, String> map = objectMapper.convertValue(value, Map.class);
|
||||
// if (jsonDataPath.getMergedFields() != null && !jsonDataPath.getMergedFields().isEmpty() && jsonDataPath.getMergedFieldName() != null && !jsonDataPath.getMergedFieldName().isEmpty()) {
|
||||
// Map<String, String> finalMap = new HashMap<>();
|
||||
// for (Map.Entry<String, String> entry : map.entrySet()) {
|
||||
// if (jsonDataPath.getMergedFields().contains(entry.getKey())) {
|
||||
// if (!finalMap.containsKey(jsonDataPath.getMergedFieldName())) {
|
||||
// finalMap.put(jsonDataPath.getMergedFieldName(), entry.getValue());
|
||||
// } else {
|
||||
// finalMap.put(jsonDataPath.getMergedFieldName(), finalMap.get(jsonDataPath.getMergedFieldName()) + " " + entry.getValue());
|
||||
// }
|
||||
// } else {
|
||||
// finalMap.put(entry.getKey(), entry.getValue());
|
||||
// }
|
||||
// }
|
||||
// values.add(finalMap);
|
||||
// } else {
|
||||
// values.add(map);
|
||||
// }
|
||||
// }
|
||||
// externalRefernceResult = new ExternalRefernceResult(values, new HashMap<>(1, 1));
|
||||
// }
|
||||
//
|
||||
// if (externalRefernceResult.getPagination().isEmpty()) {
|
||||
// externalRefernceResult.getPagination().put("pages", 1);
|
||||
// externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size());
|
||||
// }
|
||||
// return externalRefernceResult;
|
||||
// }
|
||||
// } catch (Exception exception) {
|
||||
// logger.error(exception.getMessage(), exception);
|
||||
// } //maybe print smth...
|
||||
//
|
||||
// return null;
|
||||
// }
|
||||
//
|
||||
// private List<Map<String, String>> getAllResultsFromMockUpJson(String path, String query) {
|
||||
// List<Map<String, String>> internalResults;
|
||||
// try {
|
||||
// String filePath = Paths.get(path).toUri().toURL().toString();
|
||||
// ObjectMapper mapper = new ObjectMapper();
|
||||
// internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
|
||||
// return searchListMap(internalResults, query);
|
||||
// } catch (Exception e) {
|
||||
// logger.error(e.getMessage(), e);
|
||||
// return new LinkedList<>();
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
|
||||
// List<Map<String, String>> list = new LinkedList<>();
|
||||
// for (Map<String, String> map : internalResults)
|
||||
// {
|
||||
// if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
|
||||
// list.add(map);
|
||||
// }
|
||||
// if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
|
||||
// list.add(map);
|
||||
// }
|
||||
// }
|
||||
// return list;
|
||||
// }
|
||||
//
|
||||
// private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) {
|
||||
// if(dataUrlConfiguration.getFieldsUrlConfiguration().getPid() == null)
|
||||
// return "pid";
|
||||
// else
|
||||
// return "originalId";
|
||||
// }
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPid() != null && key.equals("pid")) return "pid";
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPidTypeField() != null && key.equals("pidTypeField")) return "pidTypeField";
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPath() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getPath().replace("'",""))) return "path";
|
||||
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getHost() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getHost().replace("'",""))) return "host";
|
||||
// return null;
|
||||
// }
|
||||
//
|
||||
// private String parseBodyString(String bodyString) {
|
||||
// String finalBodyString = bodyString;
|
||||
// if (bodyString.contains("{env:")) {
|
||||
// int index = bodyString.indexOf("{env: ");
|
||||
// while (index >= 0) {
|
||||
// int endIndex = bodyString.indexOf("}", index + 6);
|
||||
// String envName = bodyString.substring(index + 6, endIndex);
|
||||
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
|
||||
// index = bodyString.indexOf("{env: ", index + 6);
|
||||
// }
|
||||
// }
|
||||
// return finalBodyString;
|
||||
// }
|
||||
//
|
||||
//}
|
||||
}
|
||||
|
||||
private String replaceLookupFieldQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
|
||||
String finalQuery = "";
|
||||
if (this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike())) return finalQuery;
|
||||
QueryConfig queryConfig = queryConfigs.stream().filter(x -> !this.conventionService.isNullOrEmpty(x.getCondition()) && externalReferenceCriteria.getLike().matches(x.getCondition()))
|
||||
.min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
|
||||
if (queryConfig != null) {
|
||||
if (queryConfig.getSeparator() != null) {
|
||||
String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
|
||||
finalQuery = queryConfig.getValue();
|
||||
for (int i = 0; i < likes.length; i++) {
|
||||
finalQuery = finalQuery.replaceAll("\\{like" + (i + 1) + "}", likes[i]);
|
||||
}
|
||||
} else {
|
||||
finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike());
|
||||
}
|
||||
|
||||
}
|
||||
return finalQuery;
|
||||
}
|
||||
|
||||
protected String replaceLookupFields(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
|
||||
if (!this.conventionService.isNullOrEmpty(path)) return path;
|
||||
String completedPath = path;
|
||||
|
||||
if (externalReferenceCriteria.getLike() != null) {
|
||||
if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().isEmpty()) {
|
||||
completedPath = completedPath.replaceAll("\\{like}", "*");
|
||||
completedPath = completedPath.replaceAll("\\{query}", "*");
|
||||
} else {
|
||||
if (completedPath.contains("{query}")) {
|
||||
completedPath = completedPath.replaceAll("\\{query}", this.replaceLookupFieldQuery(externalReferenceCriteria, queries));
|
||||
} else {
|
||||
completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike());
|
||||
}
|
||||
}
|
||||
} else {
|
||||
completedPath = completedPath.replace("{like}", "");
|
||||
}
|
||||
if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getFunderId())) {
|
||||
String[] founderParts = externalReferenceCriteria.getFunderId().split(":");
|
||||
String funderPrefix = founderParts.length > 0 ? founderParts[0] : "";
|
||||
String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", "");
|
||||
if (!funderId.isEmpty() && funderId.toCharArray()[0] == ':') {
|
||||
funderId = externalReferenceCriteria.getFunderId();
|
||||
}
|
||||
completedPath = completedPath.replace("{funderId}", funderId);
|
||||
} else if(completedPath.contains("{funderId}")){
|
||||
logger.warn("FunderId is null.");
|
||||
completedPath = completedPath.replace("{funderId}", " ");
|
||||
}
|
||||
|
||||
if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPage())) completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
|
||||
else if (!this.conventionService.isNullOrEmpty(firstPage)) completedPath = completedPath.replace("{page}", firstPage);
|
||||
else completedPath = completedPath.replace("{page}", "1");
|
||||
|
||||
completedPath = completedPath.replace("{pageSize}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPageSize()) ? externalReferenceCriteria.getPageSize() : "60");
|
||||
completedPath = completedPath.replace("{host}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getHost()) ? externalReferenceCriteria.getHost() : "");
|
||||
completedPath = completedPath.replace("{path}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPath()) ? externalReferenceCriteria.getPath() : "");
|
||||
|
||||
|
||||
return completedPath;
|
||||
}
|
||||
|
||||
private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
|
||||
String replacedPath = replaceLookupFields(apiSource.getUrl(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
|
||||
String replacedBody = replaceLookupFields(apiSource.getRequestBody(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
|
||||
|
||||
ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
|
||||
if(externalDataResult != null) {
|
||||
if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
|
||||
externalDataResult.setResults(externalDataResult.getResults().stream()
|
||||
.filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
|
||||
.collect(Collectors.toList()));
|
||||
}
|
||||
externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList());
|
||||
if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
|
||||
|
||||
Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
|
||||
if (externalDataResult.getResults().size() > maxResults)
|
||||
throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query");
|
||||
|
||||
return externalDataResult;
|
||||
}
|
||||
else {
|
||||
return new ExternalDataResult();
|
||||
}
|
||||
}
|
||||
|
||||
protected ExternalDataResult getExternalDataResults(String urlString, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, String requestBody, String auth) {
|
||||
|
||||
try {
|
||||
JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
|
||||
HttpMethod method;
|
||||
switch (apiSource.getHttpMethod()) {
|
||||
case GET -> method = HttpMethod.GET;
|
||||
case POST -> method =HttpMethod.POST;
|
||||
default -> throw new MyApplicationException("unrecognized type " + apiSource.getHttpMethod());
|
||||
}
|
||||
|
||||
ResponseEntity<String> response = this.getWebClient().method(method).uri(urlString).headers(httpHeaders -> {
|
||||
if (this.conventionService.isNullOrEmpty(apiSource.getContentType())) {
|
||||
httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(apiSource.getContentType())));
|
||||
httpHeaders.setContentType(MediaType.valueOf(apiSource.getContentType()));
|
||||
}
|
||||
if (auth != null) {
|
||||
httpHeaders.set("Authorization", auth);
|
||||
}
|
||||
}).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
|
||||
if (response == null || !response.getStatusCode().isSameCodeAs(HttpStatus.OK) || !response.hasBody() || response.getBody() == null) return null;
|
||||
|
||||
//do here all the parsing
|
||||
List<String> responseContentTypeHeader = response.getHeaders().getOrDefault("Content-Type", null);
|
||||
String responseContentType = !this.conventionService.isListNullOrEmpty(responseContentTypeHeader) && responseContentTypeHeader.getFirst() != null ? responseContentTypeHeader.getFirst() : "";
|
||||
|
||||
if (responseContentType.contains("json") ) {
|
||||
DocumentContext jsonContext = JsonPath.parse(response.getBody());
|
||||
return this.parseData(jsonContext, apiSource.getResults());
|
||||
} else {
|
||||
throw new MyApplicationException("Unsupported response type" + responseContentType);
|
||||
}
|
||||
|
||||
} catch (Exception exception) {
|
||||
logger.error(exception.getMessage(), exception);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration<ResultFieldsMappingConfiguration> resultsConfigurationEntity) {
|
||||
ExternalDataResult result = new ExternalDataResult();
|
||||
if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult();
|
||||
List<Map<String, Object>> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath());
|
||||
|
||||
result.setRawData(rawData);
|
||||
|
||||
if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult();
|
||||
|
||||
List<Map<String, String>> parsedData = new ArrayList<>();
|
||||
for (Map<String, Object> stringObjectMap: rawData){
|
||||
Map<String, String> map = new HashMap<>();
|
||||
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){
|
||||
String pathValue = field.getResponsePath();
|
||||
if (!pathValue.contains(".")){
|
||||
if (stringObjectMap.containsKey(pathValue)) {
|
||||
map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
|
||||
}
|
||||
}else {
|
||||
if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
|
||||
String value = null;
|
||||
Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
|
||||
if (fieldObj != null){
|
||||
if (fieldObj instanceof Map){
|
||||
Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
|
||||
if(o instanceof String){
|
||||
value = (String)o;
|
||||
}
|
||||
else if(o instanceof Integer){
|
||||
value = String.valueOf(o);
|
||||
}
|
||||
} else if (fieldObj instanceof List) {
|
||||
Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
|
||||
if(o instanceof String){
|
||||
value = (String)o;
|
||||
}
|
||||
else if(o instanceof Integer){
|
||||
value = String.valueOf(o);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (value != null){
|
||||
map.put(field.getCode(), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
parsedData.add(map);
|
||||
}
|
||||
result.setResults(parsedData);
|
||||
return result;
|
||||
}
|
||||
|
||||
private static String normalizeValue(Object value) {
|
||||
if (value instanceof JSONArray jsonArray) {
|
||||
|
||||
if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
|
||||
return jsonArray.getFirst().toString();
|
||||
} else {
|
||||
for (Object o : jsonArray) {
|
||||
if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
|
||||
try {
|
||||
return ((Map<String, String>) o).get("content");
|
||||
}
|
||||
catch (ClassCastException e){
|
||||
if(((Map<?, ?>) o).get("content") instanceof Integer) {
|
||||
return String.valueOf(((Map<?, ?>) o).get("content"));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if (value instanceof Map) {
|
||||
String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
|
||||
return ((Map<String, String>)value).get(key);
|
||||
}
|
||||
return value != null ? value.toString() : null;
|
||||
}
|
||||
|
||||
private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) {
|
||||
List<Map<String, String>> internalResults;
|
||||
try {
|
||||
String filePath = Paths.get(path).toUri().toURL().toString();
|
||||
ObjectMapper mapper = new ObjectMapper();
|
||||
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
|
||||
return new ExternalDataResult(searchListMap(internalResults, query));
|
||||
} catch (Exception e) {
|
||||
logger.error(e.getMessage(), e);
|
||||
return new ExternalDataResult();
|
||||
}
|
||||
}
|
||||
|
||||
private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
|
||||
List<Map<String, String>> list = new LinkedList<>();
|
||||
for (Map<String, String> map : internalResults)
|
||||
{
|
||||
if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
|
||||
list.add(map);
|
||||
}
|
||||
if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
|
||||
list.add(map);
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
private String parseBodyString(String bodyString) {
|
||||
String finalBodyString = bodyString;
|
||||
if (bodyString.contains("{env:")) {
|
||||
int index = bodyString.indexOf("{env: ");
|
||||
while (index >= 0) {
|
||||
int endIndex = bodyString.indexOf("}", index + 6);
|
||||
String envName = bodyString.substring(index + 6, endIndex);
|
||||
finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
|
||||
index = bodyString.indexOf("{env: ", index + 6);
|
||||
}
|
||||
}
|
||||
return finalBodyString;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -1,152 +0,0 @@
|
|||
//package eu.eudat.service.remotefetcher;
|
||||
//
|
||||
//import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
//import com.jayway.jsonpath.DocumentContext;
|
||||
//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
|
||||
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
||||
//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
|
||||
//import net.minidev.json.JSONArray;
|
||||
//import org.slf4j.Logger;
|
||||
//import org.slf4j.LoggerFactory;
|
||||
//
|
||||
//import java.lang.reflect.InvocationTargetException;
|
||||
//import java.lang.reflect.Method;
|
||||
//import java.util.*;
|
||||
//import java.util.stream.Collectors;
|
||||
//
|
||||
//public class RemoteFetcherUtils {
|
||||
// private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class);
|
||||
// private static final ObjectMapper mapper = new ObjectMapper();
|
||||
//
|
||||
// public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
||||
// return new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
|
||||
// new HashMap<>(1, 1));
|
||||
// }
|
||||
//
|
||||
// public static ExternalRefernceResult getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcherServiceImpl remoteFetcherService, String requestBody, String requestType, String auth) {
|
||||
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
|
||||
// new HashMap<>(1, 1));
|
||||
//
|
||||
// List<Map<String, String>> multiResults = externalRefernceResult.getResults().stream().map(result -> {
|
||||
// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria();
|
||||
// externalReferenceCriteria.setPath(result.get("path"));
|
||||
// externalReferenceCriteria.setHost(result.get("host"));
|
||||
// String replacedPath = remoteFetcherService.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalReferenceCriteria, jsonDataPath.getUrlConfiguration().getFirstpage(), jsonDataPath.getUrlConfiguration().getQueries());
|
||||
// return remoteFetcherService.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType, auth);
|
||||
// }).filter(Objects::nonNull).map(externalRefernceResult1 -> externalRefernceResult1.getResults().get(0)).collect(Collectors.toList());
|
||||
// return new ExternalRefernceResult(multiResults, new HashMap<>(1, 1));
|
||||
// }
|
||||
//
|
||||
// public static ExternalRefernceResult getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
||||
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
|
||||
// new HashMap<>(1, 1));
|
||||
// externalRefernceResult.getResults().stream().forEach(entry -> {
|
||||
// String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
|
||||
// entry.put("name", name);
|
||||
// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
|
||||
// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
|
||||
// });
|
||||
// return externalRefernceResult;
|
||||
// }
|
||||
//
|
||||
// private static List<Map<String, String>> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
|
||||
// List <Map<String, Object>> rawData = jsonContext.read(jsonDataPath.getPath());
|
||||
// List<Map<String, String>> parsedData = new ArrayList<>();
|
||||
// rawData.forEach(stringObjectMap -> {
|
||||
// parsedData.add(new LinkedHashMap<>());
|
||||
// Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> {
|
||||
// String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1);
|
||||
// Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0);
|
||||
// try {
|
||||
// String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration()));
|
||||
// if (value != null) {
|
||||
// if (field.getName().equals("pid") || field.getName().equals("pidTypeField")) {
|
||||
// String pid = null;
|
||||
// Object pidObj = stringObjectMap.get(value.split("\\.")[0]);
|
||||
// if(pidObj != null){
|
||||
// if(pidObj instanceof Map){
|
||||
// Object o = ((Map<String, Object>) pidObj).get(value.split("\\.")[1]);
|
||||
// if(o instanceof String){
|
||||
// pid = (String)o;
|
||||
// }
|
||||
// else if(o instanceof Integer){
|
||||
// pid = String.valueOf(o);
|
||||
// }
|
||||
// }
|
||||
// else if(pidObj instanceof List){
|
||||
// Object o = ((List<Map<String,?>>) pidObj).get(0).get(value.split("\\.")[1]);
|
||||
// if(o instanceof String){
|
||||
// pid = (String)o;
|
||||
// }
|
||||
// else if(o instanceof Integer){
|
||||
// pid = String.valueOf(o);
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// if(pid != null) {
|
||||
// if ((field.getName().equals("pid"))){
|
||||
// parsedData.get(parsedData.size() - 1).put("pid", pid);
|
||||
// }
|
||||
// else{
|
||||
// parsedData.get(parsedData.size() - 1).put("pidTypeField", pid);
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// value = value.replace("'", "");
|
||||
// if (value.contains(".")) {
|
||||
// String[] parts = value.split("\\.");
|
||||
// Map<String, Object> tempMap = stringObjectMap;
|
||||
// for (int i = 0; i < parts.length; i++) {
|
||||
// if (tempMap.containsKey(parts[i])) {
|
||||
// if (i + 1 < parts.length) {
|
||||
// tempMap = (Map<String, Object>) tempMap.get(parts[i]);
|
||||
// } else {
|
||||
// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(tempMap.get(parts[i]), (field.getName().equals("types") || field.getName().equals("uri"))));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// if (stringObjectMap.containsKey(value)) {
|
||||
// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri"))));
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// } catch (IllegalAccessException | InvocationTargetException e) {
|
||||
// logger.error(e.getLocalizedMessage(), e);
|
||||
// }
|
||||
// });
|
||||
// });
|
||||
// return parsedData;
|
||||
// }
|
||||
//
|
||||
// private static String normalizeValue(Object value, boolean jsonString) {
|
||||
// if (value instanceof JSONArray) {
|
||||
// if (jsonString) {
|
||||
// return ((JSONArray)value).toJSONString();
|
||||
// }
|
||||
// JSONArray jarr = (JSONArray) value;
|
||||
// if (jarr.get(0) instanceof String) {
|
||||
// return jarr.get(0).toString();
|
||||
// } else {
|
||||
// for (Object o : jarr) {
|
||||
// if ((o instanceof Map) && ((Map) o).containsKey("content")) {
|
||||
// try {
|
||||
// return ((Map<String, String>) o).get("content");
|
||||
// }
|
||||
// catch (ClassCastException e){
|
||||
// if(((Map<?, ?>) o).get("content") instanceof Integer) {
|
||||
// return String.valueOf(((Map<?, ?>) o).get("content"));
|
||||
// }
|
||||
// return null;
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// } else if (value instanceof Map) {
|
||||
// String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
|
||||
// return ((Map<String, String>)value).get(key);
|
||||
// }
|
||||
// return value != null ? value.toString() : null;
|
||||
// }
|
||||
//}
|
|
@ -1,57 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
|
||||
public class AuthenticationConfiguration {
|
||||
|
||||
private String authUrl;
|
||||
private String authMethod = "GET";
|
||||
private String authTokenPath;
|
||||
private String authRequestBody;
|
||||
private String type;
|
||||
|
||||
public String getAuthUrl() {
|
||||
return authUrl;
|
||||
}
|
||||
|
||||
@XmlElement(name = "authUrl")
|
||||
public void setAuthUrl(String authUrl) {
|
||||
this.authUrl = authUrl;
|
||||
}
|
||||
|
||||
public String getAuthMethod() {
|
||||
return authMethod;
|
||||
}
|
||||
|
||||
@XmlElement(name = "authUrlMethod")
|
||||
public void setAuthMethod(String authMethod) {
|
||||
this.authMethod = authMethod;
|
||||
}
|
||||
|
||||
public String getAuthTokenPath() {
|
||||
return authTokenPath;
|
||||
}
|
||||
|
||||
@XmlElement(name = "authTokenJpath")
|
||||
public void setAuthTokenPath(String authTokenPath) {
|
||||
this.authTokenPath = authTokenPath;
|
||||
}
|
||||
|
||||
public String getAuthRequestBody() {
|
||||
return authRequestBody;
|
||||
}
|
||||
|
||||
@XmlElement(name = "authUrlBody")
|
||||
public void setAuthRequestBody(String authRequestBody) {
|
||||
this.authRequestBody = authRequestBody;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
@XmlElement(name = "authType")
|
||||
public void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
}
|
|
@ -1,134 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
|
||||
/**
|
||||
* Created by ikalyvas on 6/29/2018.
|
||||
*/
|
||||
public class DataFieldsUrlConfiguration {
|
||||
private String id;
|
||||
private String name;
|
||||
private String pid;
|
||||
private String pidTypeField;
|
||||
private String uri;
|
||||
private String description;
|
||||
private String source;
|
||||
private String count;
|
||||
private String path;
|
||||
private String host;
|
||||
private String types;
|
||||
private String firstName;
|
||||
private String lastName;
|
||||
|
||||
public String getId() {
|
||||
return id;
|
||||
}
|
||||
|
||||
@XmlElement(name = "id")
|
||||
public void setId(String id) {
|
||||
this.id = id;
|
||||
}
|
||||
|
||||
public String getName() {
|
||||
return name;
|
||||
}
|
||||
|
||||
@XmlElement(name = "name")
|
||||
public void setName(String name) {
|
||||
this.name = name;
|
||||
}
|
||||
|
||||
public String getPid() {
|
||||
return pid;
|
||||
}
|
||||
|
||||
@XmlElement(name = "pid")
|
||||
public void setPid(String pid) {
|
||||
this.pid = pid;
|
||||
}
|
||||
|
||||
public String getPidTypeField() {
|
||||
return pidTypeField;
|
||||
}
|
||||
|
||||
@XmlElement(name = "pidTypeField")
|
||||
public void setPidTypeField(String pidTypeField) {
|
||||
this.pidTypeField = pidTypeField;
|
||||
}
|
||||
|
||||
public String getUri() {
|
||||
return uri;
|
||||
}
|
||||
|
||||
@XmlElement(name = "uri")
|
||||
public void setUri(String uri) {
|
||||
this.uri = uri;
|
||||
}
|
||||
|
||||
public String getDescription() {
|
||||
return description;
|
||||
}
|
||||
|
||||
@XmlElement(name = "description")
|
||||
public void setDescription(String description) {
|
||||
this.description = description;
|
||||
}
|
||||
|
||||
public String getSource() {
|
||||
return source;
|
||||
}
|
||||
|
||||
@XmlElement(name = "source")
|
||||
public void setSource(String source) {
|
||||
this.source = source;
|
||||
}
|
||||
|
||||
public String getCount() {
|
||||
return count;
|
||||
}
|
||||
|
||||
@XmlElement(name = "count")
|
||||
public void setCount(String count) {
|
||||
this.count = count;
|
||||
}
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
@XmlElement(name = "path")
|
||||
public void setPath(String path) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
public String getHost() {
|
||||
return host;
|
||||
}
|
||||
@XmlElement(name = "host")
|
||||
public void setHost(String host) {
|
||||
this.host = host;
|
||||
}
|
||||
|
||||
@XmlElement(name = "types")
|
||||
public String getTypes() {
|
||||
return types;
|
||||
}
|
||||
public void setTypes(String types) {
|
||||
this.types = types;
|
||||
}
|
||||
|
||||
@XmlElement(name = "firstName")
|
||||
public String getFirstName() {
|
||||
return firstName;
|
||||
}
|
||||
public void setFirstName(String firstName) {
|
||||
this.firstName = firstName;
|
||||
}
|
||||
|
||||
@XmlElement(name = "lastName")
|
||||
public String getLastName() {
|
||||
return lastName;
|
||||
}
|
||||
public void setLastName(String lastName) {
|
||||
this.lastName = lastName;
|
||||
}
|
||||
}
|
|
@ -1,82 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Created by ikalyvas on 6/29/2018.
|
||||
*/
|
||||
public class DataUrlConfiguration {
|
||||
private String path;
|
||||
private DataFieldsUrlConfiguration fieldsUrlConfiguration;
|
||||
private UrlConfiguration urlConfiguration;
|
||||
private String parseClass;
|
||||
private String parseField;
|
||||
private List<String> mergedFields;
|
||||
private String mergedFieldName;
|
||||
|
||||
public String getPath() {
|
||||
return path;
|
||||
}
|
||||
|
||||
@XmlElement(name = "path")
|
||||
public void setPath(String path) {
|
||||
this.path = path;
|
||||
}
|
||||
|
||||
public DataFieldsUrlConfiguration getFieldsUrlConfiguration() {
|
||||
return fieldsUrlConfiguration;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fields")
|
||||
public void setFieldsUrlConfiguration(DataFieldsUrlConfiguration fieldsUrlConfiguration) {
|
||||
this.fieldsUrlConfiguration = fieldsUrlConfiguration;
|
||||
}
|
||||
|
||||
public UrlConfiguration getUrlConfiguration() {
|
||||
return urlConfiguration;
|
||||
}
|
||||
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrlConfiguration(UrlConfiguration urlConfiguration) {
|
||||
this.urlConfiguration = urlConfiguration;
|
||||
}
|
||||
|
||||
public String getParseClass() {
|
||||
return parseClass;
|
||||
}
|
||||
|
||||
@XmlElement(name = "parse-class")
|
||||
public void setParseClass(String parseClass) {
|
||||
this.parseClass = parseClass;
|
||||
}
|
||||
|
||||
public String getParseField() {
|
||||
return parseField;
|
||||
}
|
||||
|
||||
@XmlElement(name = "parse-field")
|
||||
public void setParseField(String parseField) {
|
||||
this.parseField = parseField;
|
||||
}
|
||||
|
||||
public List<String> getMergedFields() {
|
||||
return mergedFields;
|
||||
}
|
||||
|
||||
@XmlElementWrapper(name = "merge-fields")
|
||||
@XmlElement(name = "field")
|
||||
public void setMergedFields(List<String> mergedFields) {
|
||||
this.mergedFields = mergedFields;
|
||||
}
|
||||
|
||||
public String getMergedFieldName() {
|
||||
return mergedFieldName;
|
||||
}
|
||||
|
||||
@XmlElement(name = "merge-field-name")
|
||||
public void setMergedFieldName(String mergedFieldName) {
|
||||
this.mergedFieldName = mergedFieldName;
|
||||
}
|
||||
}
|
|
@ -17,153 +17,8 @@ public class ExternalUrls implements Serializable {
|
|||
|
||||
Long maxresults;
|
||||
|
||||
GrantUrls grants;
|
||||
ProjectUrls projects;
|
||||
RegistryUrls registries;
|
||||
RepositoryUrls repositories;
|
||||
PubRepositoryUrls pubRepositories;
|
||||
JournalUrls journals;
|
||||
TaxonomiesUrls taxonomies;
|
||||
PublicationsUrls publications;
|
||||
ServiceUrls services;
|
||||
ResearcherUrls researchers;
|
||||
OrganisationUrls organisations;
|
||||
DatasetUrls datasets;
|
||||
/*TagUrls tags;*/
|
||||
FunderUrls funders;
|
||||
LicenseUrls licenses;
|
||||
ValidationUrls validations;
|
||||
Map<String, PrefillingConfig> prefillings;
|
||||
|
||||
|
||||
public RegistryUrls getRegistries() {
|
||||
return registries;
|
||||
}
|
||||
|
||||
@XmlElement(name = "registries")
|
||||
public void setRegistries(RegistryUrls registries) {
|
||||
this.registries = registries;
|
||||
}
|
||||
|
||||
|
||||
public RepositoryUrls getRepositories() {
|
||||
return repositories;
|
||||
}
|
||||
|
||||
@XmlElement(name = "repositories")
|
||||
public void setRepositories(RepositoryUrls repositories) {
|
||||
this.repositories = repositories;
|
||||
}
|
||||
|
||||
public PubRepositoryUrls getPubRepositories() {
|
||||
return pubRepositories;
|
||||
}
|
||||
|
||||
public void setPubRepositories(PubRepositoryUrls pubRepositories) {
|
||||
this.pubRepositories = pubRepositories;
|
||||
}
|
||||
|
||||
public void setJournals(JournalUrls journals) {
|
||||
this.journals = journals;
|
||||
}
|
||||
|
||||
public JournalUrls getJournals() {
|
||||
return journals;
|
||||
}
|
||||
|
||||
public JournalUrls getJournal() {
|
||||
return journals;
|
||||
}
|
||||
@XmlElement(name = "journals")
|
||||
public void setJournal(JournalUrls journal) {
|
||||
this.journals = journal;
|
||||
}
|
||||
|
||||
public TaxonomiesUrls getTaxonomies() {
|
||||
return taxonomies;
|
||||
}
|
||||
@XmlElement(name = "taxonomies")
|
||||
public void setTaxonomies(TaxonomiesUrls taxonomies) {
|
||||
this.taxonomies = taxonomies;
|
||||
}
|
||||
|
||||
public PublicationsUrls getPublications() {
|
||||
return publications;
|
||||
}
|
||||
|
||||
public void setPublications(PublicationsUrls publications) {
|
||||
this.publications = publications;
|
||||
}
|
||||
|
||||
public ServiceUrls getServices() {
|
||||
return services;
|
||||
}
|
||||
|
||||
@XmlElement(name = "services")
|
||||
public void setServices(ServiceUrls services) {
|
||||
this.services = services;
|
||||
}
|
||||
|
||||
|
||||
public ResearcherUrls getResearchers() {
|
||||
return researchers;
|
||||
}
|
||||
|
||||
@XmlElement(name = "researchers")
|
||||
public void setResearchers(ResearcherUrls researchers) {
|
||||
this.researchers = researchers;
|
||||
}
|
||||
|
||||
|
||||
/*public TagUrls getTags() {
|
||||
return tags;
|
||||
}
|
||||
|
||||
@XmlElement(name = "tags")
|
||||
public void setTags(TagUrls tags) {
|
||||
this.tags = tags;
|
||||
}*/
|
||||
|
||||
|
||||
public OrganisationUrls getOrganisations() {
|
||||
return organisations;
|
||||
}
|
||||
|
||||
@XmlElement(name = "organisations")
|
||||
public void setOrganisations(OrganisationUrls organisations) {
|
||||
this.organisations = organisations;
|
||||
}
|
||||
|
||||
|
||||
public GrantUrls getGrants() {
|
||||
return grants;
|
||||
}
|
||||
|
||||
@XmlElement(name = "grants")
|
||||
public void setGrants(GrantUrls grants) {
|
||||
this.grants = grants;
|
||||
}
|
||||
|
||||
|
||||
public ProjectUrls getProjects() {
|
||||
return projects;
|
||||
}
|
||||
|
||||
@XmlElement(name = "projects")
|
||||
public void setProjects(ProjectUrls projects) {
|
||||
this.projects = projects;
|
||||
}
|
||||
|
||||
|
||||
public FunderUrls getFunders() {
|
||||
return funders;
|
||||
}
|
||||
|
||||
public void setFunders(FunderUrls funders) {
|
||||
this.funders = funders;
|
||||
}
|
||||
|
||||
|
||||
public Long getMaxresults() {
|
||||
return maxresults;
|
||||
}
|
||||
|
@ -173,34 +28,6 @@ public class ExternalUrls implements Serializable {
|
|||
this.maxresults = maxresults;
|
||||
}
|
||||
|
||||
|
||||
public DatasetUrls getDatasets() {
|
||||
return datasets;
|
||||
}
|
||||
|
||||
@XmlElement(name = "datasets")
|
||||
public void setDatasets(DatasetUrls datasets) {
|
||||
this.datasets = datasets;
|
||||
}
|
||||
|
||||
public LicenseUrls getLicenses() {
|
||||
return licenses;
|
||||
}
|
||||
|
||||
@XmlElement(name = "licenses")
|
||||
public void setLicenses(LicenseUrls licenses) {
|
||||
this.licenses = licenses;
|
||||
}
|
||||
|
||||
public ValidationUrls getValidations() {
|
||||
return validations;
|
||||
}
|
||||
|
||||
@XmlElement(name = "validators")
|
||||
public void setValidations(ValidationUrls validations) {
|
||||
this.validations = validations;
|
||||
}
|
||||
|
||||
public Map<String, PrefillingConfig> getPrefillings() {
|
||||
return prefillings;
|
||||
}
|
||||
|
|
|
@ -1,48 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
|
||||
public class QueryConfig {
|
||||
|
||||
private String condition;
|
||||
private String separator;
|
||||
private String value;
|
||||
private Integer ordinal;
|
||||
|
||||
|
||||
public String getCondition() {
|
||||
return condition;
|
||||
}
|
||||
|
||||
@XmlElement(name = "condition")
|
||||
public void setCondition(String condition) {
|
||||
this.condition = condition;
|
||||
}
|
||||
|
||||
public String getSeparator() {
|
||||
return separator;
|
||||
}
|
||||
|
||||
@XmlElement(name = "separator")
|
||||
public void setSeparator(String separator) {
|
||||
this.separator = separator;
|
||||
}
|
||||
|
||||
public String getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
@XmlElement(name = "value")
|
||||
public void setValue(String value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public Integer getOrdinal() {
|
||||
return ordinal;
|
||||
}
|
||||
|
||||
@XmlElement(name = "ordinal")
|
||||
public void setOrdinal(Integer ordinal) {
|
||||
this.ordinal = ordinal;
|
||||
}
|
||||
}
|
|
@ -1,34 +1,23 @@
|
|||
package eu.eudat.service.remotefetcher.config;
|
||||
|
||||
|
||||
import eu.eudat.commons.enums.ReferenceTypeSourceType;
|
||||
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class UrlConfiguration {
|
||||
public class UrlConfiguration implements SourceBaseConfiguration {
|
||||
|
||||
private String key;
|
||||
private String label;
|
||||
private Integer ordinal;
|
||||
private String url;
|
||||
private boolean dataInListing;
|
||||
private DataUrlConfiguration data;
|
||||
private String type;
|
||||
private String paginationPath;
|
||||
private String contentType;
|
||||
private String funderQuery;
|
||||
private String firstpage;
|
||||
private String requestType = "GET";
|
||||
private String requestBody = "";
|
||||
private String filterType = "remote";
|
||||
private AuthenticationConfiguration auth;
|
||||
|
||||
private List<QueryConfig> queries;
|
||||
private ReferenceTypeSourceType type;
|
||||
|
||||
public String getKey() {
|
||||
return key;
|
||||
}
|
||||
@XmlElement(name = "key")
|
||||
|
||||
public void setKey(String key) {
|
||||
this.key = key;
|
||||
}
|
||||
|
@ -36,121 +25,24 @@ public class UrlConfiguration {
|
|||
public String getLabel() {
|
||||
return label;
|
||||
}
|
||||
@XmlElement(name = "label")
|
||||
|
||||
public void setLabel(String label) {
|
||||
this.label = label;
|
||||
}
|
||||
|
||||
public String getUrl() {
|
||||
return url;
|
||||
}
|
||||
@XmlElement(name = "url")
|
||||
public void setUrl(String url) {
|
||||
this.url = url;
|
||||
}
|
||||
|
||||
public boolean isDataInListing() {
|
||||
return dataInListing;
|
||||
}
|
||||
@XmlElement(name = "dataInListing")
|
||||
public void setDataInListing(boolean dataInListing) {
|
||||
this.dataInListing = dataInListing;
|
||||
}
|
||||
|
||||
public Integer getOrdinal() {
|
||||
return ordinal;
|
||||
}
|
||||
@XmlElement(name = "ordinal")
|
||||
|
||||
public void setOrdinal(Integer ordinal) {
|
||||
this.ordinal = ordinal;
|
||||
}
|
||||
|
||||
public DataUrlConfiguration getData() {
|
||||
return data;
|
||||
}
|
||||
@XmlElement(name = "data")
|
||||
public void setData(DataUrlConfiguration data) {
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public String getPaginationPath() {
|
||||
return paginationPath;
|
||||
}
|
||||
@XmlElement(name = "paginationpath")
|
||||
public void setPaginationPath(String paginationPath) {
|
||||
this.paginationPath = paginationPath;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
public ReferenceTypeSourceType getType() {
|
||||
return type;
|
||||
}
|
||||
@XmlElement(name = "type")
|
||||
public void setType(String type) {
|
||||
|
||||
public void setType(ReferenceTypeSourceType type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
public String getContentType() {
|
||||
return contentType;
|
||||
}
|
||||
@XmlElement(name = "contenttype")
|
||||
public void setContentType(String contentType) {
|
||||
this.contentType = contentType;
|
||||
}
|
||||
|
||||
public String getFunderQuery() {
|
||||
return funderQuery;
|
||||
}
|
||||
@XmlElement(name = "funderQuery")
|
||||
public void setFunderQuery(String funderQuery) {
|
||||
this.funderQuery = funderQuery;
|
||||
}
|
||||
|
||||
public String getFirstpage() {
|
||||
return firstpage;
|
||||
}
|
||||
@XmlElement(name = "firstPage")
|
||||
public void setFirstpage(String firstpage) {
|
||||
this.firstpage = firstpage;
|
||||
}
|
||||
|
||||
public String getRequestType() {
|
||||
return requestType;
|
||||
}
|
||||
@XmlElement(name = "request")
|
||||
public void setRequestType(String requestType) {
|
||||
this.requestType = requestType != null ? requestType : "GET";
|
||||
}
|
||||
public String getRequestBody() {
|
||||
return requestBody;
|
||||
}
|
||||
@XmlElement(name = "requestBody")
|
||||
public void setRequestBody(String requestBody) {
|
||||
this.requestBody = requestBody != null ? requestBody : "";
|
||||
}
|
||||
public String getFilterType() {
|
||||
return filterType;
|
||||
}
|
||||
@XmlElement(name = "filterType")
|
||||
public void setFilterType(String filterType) {
|
||||
this.filterType = filterType;
|
||||
}
|
||||
|
||||
public List<QueryConfig> getQueries() {
|
||||
return queries;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "query")
|
||||
public void setQueries(List<QueryConfig> queries) {
|
||||
this.queries = queries;
|
||||
}
|
||||
|
||||
public AuthenticationConfiguration getAuth() {
|
||||
return auth;
|
||||
}
|
||||
|
||||
@XmlElement(name="authentication")
|
||||
public void setAuth(AuthenticationConfiguration auth) {
|
||||
this.auth = auth;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,18 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
|
||||
|
||||
public interface AuthenticationConfiguration {
|
||||
|
||||
Boolean getEnabled();
|
||||
|
||||
String getAuthUrl();
|
||||
|
||||
ReferenceTypeExternalApiHTTPMethodType getAuthMethod();
|
||||
|
||||
String getAuthTokenPath();
|
||||
|
||||
String getAuthRequestBody();
|
||||
|
||||
String getType();
|
||||
}
|
|
@ -1,33 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
public class DatasetUrls extends GenericUrls{
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
}
|
|
@ -1,33 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class FunderUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
}
|
|
@ -1,31 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
public class GeneralUrls extends GenericUrls {
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public GeneralUrls() {
|
||||
this.urls = new ArrayList<>();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@Override
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
}
|
|
@ -1,12 +1,13 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public abstract class GenericUrls {
|
||||
|
||||
public abstract List<UrlConfiguration> getUrls();
|
||||
public abstract List<SourceBaseConfiguration> getUrls();
|
||||
public abstract FetchStrategy getFetchMode();
|
||||
}
|
||||
|
||||
|
||||
|
|
|
@ -1,35 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
public class GrantUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,34 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class JournalUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,33 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
|
||||
public class LicenseUrls extends GenericUrls{
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
}
|
|
@ -1,34 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class OrganisationUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
}
|
|
@ -20,8 +20,8 @@ public class PrefillingSearch extends GenericUrls {
|
|||
}
|
||||
|
||||
@Override
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
List<UrlConfiguration> urls = new ArrayList<>();
|
||||
public List<SourceBaseConfiguration> getUrls() {
|
||||
List<SourceBaseConfiguration> urls = new ArrayList<>();
|
||||
urls.add(urlConfig);
|
||||
return urls;
|
||||
}
|
||||
|
|
|
@ -1,33 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class ProjectUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
}
|
|
@ -1,34 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class PubRepositoryUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,34 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class PublicationsUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
public interface QueryConfig {
|
||||
String getCondition();
|
||||
|
||||
String getSeparator();
|
||||
|
||||
String getValue();
|
||||
|
||||
Integer getOrdinal();
|
||||
}
|
|
@ -1,35 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class RegistryUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -1,34 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class RepositoryUrls extends GenericUrls {
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,35 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class ResearcherUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,6 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
public interface ResultFieldsMappingConfiguration {
|
||||
String getResponsePath();
|
||||
String getCode();
|
||||
}
|
|
@ -0,0 +1,9 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface ResultsConfiguration <T extends ResultFieldsMappingConfiguration> {
|
||||
String getResultsArrayPath();
|
||||
|
||||
List<T> getFieldsMapping();
|
||||
}
|
|
@ -1,35 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class ServiceUrls extends GenericUrls{
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.commons.enums.ReferenceTypeSourceType;
|
||||
|
||||
public interface SourceBaseConfiguration {
|
||||
String getKey();
|
||||
String getLabel();
|
||||
Integer getOrdinal();
|
||||
ReferenceTypeSourceType getType();
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface SourceExternalApiConfiguration<RsConfig extends ResultsConfiguration<? extends ResultFieldsMappingConfiguration>, AuthConfig extends AuthenticationConfiguration, QConfig extends QueryConfig> extends SourceBaseConfiguration {
|
||||
String getUrl();
|
||||
|
||||
RsConfig getResults();
|
||||
|
||||
String getPaginationPath();
|
||||
|
||||
String getContentType();
|
||||
|
||||
String getFirstPage();
|
||||
|
||||
ReferenceTypeExternalApiHTTPMethodType getHttpMethod();
|
||||
|
||||
String getRequestBody();
|
||||
|
||||
String getFilterType();
|
||||
|
||||
AuthConfig getAuth();
|
||||
|
||||
List<QConfig> getQueries();
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
public interface SourceStaticOptionConfiguration <Option extends StaticOption> extends SourceBaseConfiguration {
|
||||
List<Option> getOptions();
|
||||
}
|
|
@ -0,0 +1,6 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
public interface StaticOption {
|
||||
String getCode();
|
||||
String getValue();
|
||||
}
|
|
@ -1,35 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* Created by ikalyvas on 7/9/2018.
|
||||
*/
|
||||
public class TagUrls extends GenericUrls{
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
}
|
|
@ -1,34 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
import java.util.List;
|
||||
|
||||
public class TaxonomiesUrls extends GenericUrls {
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
}
|
|
@ -1,35 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.config.entities;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
||||
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
||||
import jakarta.xml.bind.annotation.XmlElement;
|
||||
import jakarta.xml.bind.annotation.XmlElementWrapper;
|
||||
|
||||
public class ValidationUrls extends GenericUrls {
|
||||
|
||||
List<UrlConfiguration> urls;
|
||||
FetchStrategy fetchMode;
|
||||
|
||||
public List<UrlConfiguration> getUrls() {
|
||||
return urls;
|
||||
}
|
||||
|
||||
@XmlElementWrapper
|
||||
@XmlElement(name = "urlConfig")
|
||||
public void setUrls(List<UrlConfiguration> urls) {
|
||||
this.urls = urls;
|
||||
}
|
||||
|
||||
public FetchStrategy getFetchMode() {
|
||||
return fetchMode;
|
||||
}
|
||||
|
||||
@XmlElement(name = "fetchMode")
|
||||
public void setFetchMode(FetchStrategy fetchMode) {
|
||||
this.fetchMode = fetchMode;
|
||||
}
|
||||
|
||||
|
||||
}
|
|
@ -0,0 +1,46 @@
|
|||
package eu.eudat.service.remotefetcher.models;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class ExternalDataResult {
|
||||
List<Map<String, String>> results;
|
||||
List<Map<String, Object>> rawData;
|
||||
|
||||
public ExternalDataResult() {
|
||||
this.results = new ArrayList<>();
|
||||
}
|
||||
|
||||
public ExternalDataResult(List<Map<String, String>> results) {
|
||||
this.results = results;
|
||||
}
|
||||
|
||||
public List<Map<String, String>> getResults() {
|
||||
return results;
|
||||
}
|
||||
|
||||
public void setResults(List<Map<String, String>> results) {
|
||||
this.results = results;
|
||||
}
|
||||
|
||||
public List<Map<String, Object>> getRawData() {
|
||||
return rawData;
|
||||
}
|
||||
|
||||
public void setRawData(List<Map<String, Object>> rawData) {
|
||||
this.rawData = rawData;
|
||||
}
|
||||
|
||||
public void addAll(ExternalDataResult other){
|
||||
if (other == null) return;
|
||||
if (other.getResults() != null){
|
||||
if (this.getResults() == null) this.setResults(new ArrayList<>());
|
||||
this.getResults().addAll(other.getResults());
|
||||
}
|
||||
if (other.getRawData() != null){
|
||||
if (this.getRawData() == null) this.setRawData(new ArrayList<>());
|
||||
this.getRawData().addAll(other.getRawData());
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,37 +0,0 @@
|
|||
package eu.eudat.service.remotefetcher.models;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
public class ExternalRefernceResult {
|
||||
List<Map<String, String>> results;
|
||||
Map<String, Integer> pagination;
|
||||
|
||||
public ExternalRefernceResult() {
|
||||
this.results = new ArrayList<>();
|
||||
this.pagination = new HashMap<>();
|
||||
}
|
||||
|
||||
public ExternalRefernceResult(List<Map<String, String>> results, Map<String, Integer> pagination) {
|
||||
this.results = results;
|
||||
this.pagination = pagination;
|
||||
}
|
||||
|
||||
public List<Map<String, String>> getResults() {
|
||||
return results;
|
||||
}
|
||||
|
||||
public void setResults(List<Map<String, String>> results) {
|
||||
this.results = results;
|
||||
}
|
||||
|
||||
public Map<String, Integer> getPagination() {
|
||||
return pagination;
|
||||
}
|
||||
|
||||
public void setPagination(Map<String, Integer> pagination) {
|
||||
this.pagination = pagination;
|
||||
}
|
||||
}
|
|
@ -11,7 +11,6 @@ import eu.eudat.model.censorship.ReferenceCensor;
|
|||
import eu.eudat.model.persist.ReferencePersist;
|
||||
import eu.eudat.model.result.QueryResult;
|
||||
import eu.eudat.query.ReferenceQuery;
|
||||
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
|
||||
import eu.eudat.query.lookup.ReferenceLookup;
|
||||
import eu.eudat.query.lookup.ReferenceSearchLookup;
|
||||
import eu.eudat.service.reference.ReferenceService;
|
||||
|
@ -95,12 +94,12 @@ public class ReferenceController {
|
|||
|
||||
|
||||
@PostMapping("search")
|
||||
public List<Reference> searchReferenceWithDefinition(@RequestBody ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
|
||||
public List<Reference> searchReferenceWithDefinition(@RequestBody ReferenceSearchLookup lookup) throws MyNotFoundException {
|
||||
logger.debug("search with db definition {}", Reference.class.getSimpleName());
|
||||
|
||||
this.censorFactory.censor(ReferenceCensor.class).censor(lookup.getProject(), null);
|
||||
|
||||
List<Reference> references = this.referenceService.searchReferenceWithDefinition(lookup);
|
||||
List<Reference> references = this.referenceService.searchReferenceData(lookup);
|
||||
|
||||
this.auditService.track(AuditableAction.Reference_Search, "lookup", lookup);
|
||||
|
||||
|
|
Loading…
Reference in New Issue