MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

reference fetcher changes

This commit is contained in:
Efstratios Giannopoulos 2024-02-12 17:43:41 +02:00
parent 87dd56ff2c
commit 957c1fad58
54 changed files with 923 additions and 2591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/AuthenticationConfigurationEntity.java
View File

@ -1,9 +1,10 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.service.remotefetcher.config.entities.AuthenticationConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
public class AuthenticationConfigurationEntity { public class AuthenticationConfigurationEntity implements AuthenticationConfiguration {
private Boolean enabled; private Boolean enabled;
private String authUrl; private String authUrl;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/QueryConfigEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.QueryConfig;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
public class QueryConfigEntity { public class QueryConfigEntity implements QueryConfig {
private String condition; private String condition;
private String separator; private String separator;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceBaseConfigurationEntity.java
View File

@ -1,12 +1,13 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeSourceType; import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public abstract class ReferenceTypeSourceBaseConfigurationEntity { public abstract class ReferenceTypeSourceBaseConfigurationEntity implements SourceBaseConfiguration {
private String key; private String key;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceExternalApiConfigurationEntity.java
View File

@ -2,11 +2,12 @@ package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.service.remotefetcher.config.entities.SourceExternalApiConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity { public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceExternalApiConfiguration<ResultsConfigurationEntity, AuthenticationConfigurationEntity, QueryConfigEntity> {
private String url; private String url;
private ResultsConfigurationEntity results; private ResultsConfigurationEntity results;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceStaticOptionConfigurationEntity.java
View File

@ -1,11 +1,12 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.SourceStaticOptionConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity { public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceStaticOptionConfiguration<ReferenceTypeStaticOptionEntity> {
List<ReferenceTypeStaticOptionEntity> options; List<ReferenceTypeStaticOptionEntity> options;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeStaticOptionEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.StaticOption;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
public class ReferenceTypeStaticOptionEntity { public class ReferenceTypeStaticOptionEntity implements StaticOption {
private String code; private String code;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultFieldsMappingConfigurationEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.ResultFieldsMappingConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
public class ResultFieldsMappingConfigurationEntity { public class ResultFieldsMappingConfigurationEntity implements ResultFieldsMappingConfiguration {
private String code; private String code;
private String responsePath; private String responsePath;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultsConfigurationEntity.java
View File

@ -1,11 +1,12 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.ResultsConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public class ResultsConfigurationEntity { public class ResultsConfigurationEntity implements ResultsConfiguration<ResultFieldsMappingConfigurationEntity> {
private String resultsArrayPath; private String resultsArrayPath;
private List<ResultFieldsMappingConfigurationEntity> fieldsMapping; private List<ResultFieldsMappingConfigurationEntity> fieldsMapping;

58
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/DefinitionSearchBuilder.java
View File

@ -1,58 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Definition;
//import gr.cite.tools.data.builder.BuilderFactory;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class DefinitionSearchBuilder extends BaseBuilder<Definition, Map<String, String>> {
//
// private final BuilderFactory builderFactory;
// private final ReferenceTypeProperties properties;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public DefinitionSearchBuilder(
// ConventionService conventionService, BuilderFactory builderFactory, ReferenceTypeProperties properties) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(DefinitionSearchBuilder.class)));
// this.builderFactory = builderFactory;
// this.properties = properties;
// }
//
// public DefinitionSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Definition> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Definition> models = new ArrayList<>();
// //for (Map<String, String> d : data) {
// Definition m = new Definition();
// m.setFields(this.builderFactory.builder(FieldSearchBuilder.class).authorize(this.authorize).build(null, data));
// models.add(m);
// //}
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//}

88
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/FieldSearchBuilder.java
View File

@ -1,88 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.commons.enums.ReferenceFieldDataType;
//import eu.eudat.configurations.referencetype.ReferenceTypeField;
//import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Field;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class FieldSearchBuilder extends BaseBuilder<Field, Map<String, String>> {
//
// private final ReferenceTypeProperties properties;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public FieldSearchBuilder(
// ConventionService conventionService, ReferenceTypeProperties properties) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(FieldSearchBuilder.class)));
// this.properties = properties;
// }
//
// public FieldSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Field> build(FieldSet fields, List< Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Field> models = new ArrayList<>();
// for (Map<String, String> d : data) {
//
// ReferenceType referenceType = ReferenceType.valueOf(d.getOrDefault("referenceType", null));
// List<ReferenceTypeField> typeFields = this.getPropertiesFields(referenceType);
// if (typeFields.isEmpty()){
// return new ArrayList<>();
// }
//
// for (ReferenceTypeField typeField: typeFields){
// Field m = new Field();
// m.setCode(typeField.getCode());
// m.setDataType(ReferenceFieldDataType.valueOf(typeField.getDataType()));
// m.setValue(d.getOrDefault(typeField.getCode(), null));
// models.add(m);
// }
// }
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//
// private List<ReferenceTypeField> getPropertiesFields(ReferenceType referenceType){
// switch (referenceType) {
// case Taxonomies: return properties.getTaxonomy().get("fields");
// case Licenses: return properties.getLicence().get("fields");
// case Publications: return properties.getPublication().get("fields");
// case Journals: return properties.getJournal().get("fields");
// case PubRepositories: return properties.getPubRepository().get("fields");
// case DataRepositories: return properties.getDataRepository().get("fields");
// case Registries: return properties.getRegistry().get("fields");
// case Services: return properties.getService().get("fields");
// case Organizations: return properties.getOrganisation().get("fields");
// case Datasets: return properties.getDataset().get("fields");
// case Funder: return properties.getFunder().get("fields");
// case Grants: return properties.getGrant().get("fields");
// case Project: return properties.getProject().get("fields");
// case Researcher: return properties.getResearcher().get("fields");
// default: return null;
// }
// }
//}

96
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/ReferenceSearchBuilder.java
View File

@ -1,96 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.commons.enums.ReferenceSourceType;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.Reference;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Definition;
//import eu.eudat.model.referencedefinition.Field;
//import gr.cite.tools.data.builder.BuilderFactory;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class ReferenceSearchBuilder extends BaseBuilder<Reference, Map<String, String>> {
//
// private final BuilderFactory builderFactory;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public ReferenceSearchBuilder(
// ConventionService conventionService,
// BuilderFactory builderFactory) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(ReferenceSearchBuilder.class)));
// this.builderFactory = builderFactory;
// }
//
// public ReferenceSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Reference> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Reference> models = new ArrayList<>();
// for (Map<String, String> d : data) {
// Reference m = new Reference();
// if (d.containsKey("id")) m.setId(UUID.fromString(d.getOrDefault("id", null)));
// m.setLabel(d.getOrDefault("name", null));
// m.setSourceType(ReferenceSourceType.External);
// String type = d.getOrDefault("referenceType", null);
// if (type != null) m.setType(ReferenceType.valueOf(type));
//
// Definition definition = this.builderFactory.builder(DefinitionSearchBuilder.class).authorize(this.authorize).build(null, d);
// m.setDefinition(definition);
//
// switch (m.getType()){
// case Researcher:
// case Organizations:
// case Funder:
// case Grants:
// case DataRepositories:
// case PubRepositories:
// case Journals:
// case Datasets:
// case Registries:
// case Services:{
// for (Field field : definition.getFields()) {
// if (field.getCode().equals("pid") && field.getValue() != null) {
// if (d.containsKey("key")) {
// m.setReference(d.get("key") + ":" + field.getValue());
// }
// }
// if (field.getCode().equals("tag") && field.getValue() != null){
// m.setSource(field.getValue());
// }
// }
// }
// }
//
// m.setDescription(d.getOrDefault("description", null));
// m.setAbbreviation(d.getOrDefault("abbreviation", null));
//
// models.add(m);
//
// }
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//
//}

10
dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java
View File

@ -20,6 +20,7 @@ import eu.eudat.model.descriptionproperties.PropertyDefinitionFieldSetItem;
import eu.eudat.model.persist.DescriptionProfilingRequest; import eu.eudat.model.persist.DescriptionProfilingRequest;
import eu.eudat.model.persist.DescriptionProfilingWithDataRequest; import eu.eudat.model.persist.DescriptionProfilingWithDataRequest;
import eu.eudat.service.remotefetcher.ExternalUrlConfigProvider; import eu.eudat.service.remotefetcher.ExternalUrlConfigProvider;
import eu.eudat.service.remotefetcher.RemoteFetcherService;
import eu.eudat.service.remotefetcher.config.entities.*; import eu.eudat.service.remotefetcher.config.entities.*;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.builder.BuilderFactory;
@ -66,6 +67,7 @@ public class PrefillingServiceImpl implements PrefillingService {
private final ExternalUrlConfigProvider externalUrlConfigProvider; private final ExternalUrlConfigProvider externalUrlConfigProvider;
private final XmlHandlingService xmlHandlingService; private final XmlHandlingService xmlHandlingService;
private final ValidatorFactory validatorFactory; private final ValidatorFactory validatorFactory;
private final RemoteFetcherService remoteFetcherService;
@Autowired @Autowired
public PrefillingServiceImpl( public PrefillingServiceImpl(
EntityManager entityManager, EntityManager entityManager,
@ -75,8 +77,9 @@ public class PrefillingServiceImpl implements PrefillingService {
MessageSource messageSource, MessageSource messageSource,
JsonHandlingService jsonHandlingService, JsonHandlingService jsonHandlingService,
ExternalUrlConfigProvider externalUrlConfigProvider, ExternalUrlConfigProvider externalUrlConfigProvider,
XmlHandlingService xmlHandlingService, XmlHandlingService xmlHandlingService,
ValidatorFactory validatorFactory) { ValidatorFactory validatorFactory,
RemoteFetcherService remoteFetcherService) {
this.entityManager = entityManager; this.entityManager = entityManager;
this.builderFactory = builderFactory; this.builderFactory = builderFactory;
this.queryFactory = queryFactory; this.queryFactory = queryFactory;
@ -86,6 +89,7 @@ public class PrefillingServiceImpl implements PrefillingService {
this.externalUrlConfigProvider = externalUrlConfigProvider; this.externalUrlConfigProvider = externalUrlConfigProvider;
this.xmlHandlingService = xmlHandlingService; this.xmlHandlingService = xmlHandlingService;
this.validatorFactory = validatorFactory; this.validatorFactory = validatorFactory;
this.remoteFetcherService = remoteFetcherService;
} }
@Override @Override
@ -98,7 +102,7 @@ public class PrefillingServiceImpl implements PrefillingService {
List<Map<String, String>> map; List<Map<String, String>> map;
Map<String, PrefillingConfig> prefillingConfigs = this.externalUrlConfigProvider.getExternalUrls().getPrefillings(); Map<String, PrefillingConfig> prefillingConfigs = this.externalUrlConfigProvider.getExternalUrls().getPrefillings();
// for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {//TODO new reference logic // for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {//TODO new reference logic
// map = remoteFetcherService.getExternalGeneric(externalReferenceCriteria, prefillingConfig.getPrefillingSearch()); // map = remoteFetcherService.getExternalData(prefillingConfig.getPrefillingSearch().getUrls(), externalReferenceCriteria, prefillingConfig.getPrefillingSearch().getFetchMode());
// prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList()); // prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList());
// if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) { // if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) {
// List<Map<String, Object>> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch()); // List<Map<String, Object>> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());

6
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java
View File

@ -2,10 +2,12 @@ package eu.eudat.service.reference;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.Reference; import eu.eudat.model.Reference;
import eu.eudat.model.persist.ReferencePersist; import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup; import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException; import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException; import gr.cite.tools.exception.MyNotFoundException;
@ -24,5 +26,5 @@ public interface ReferenceService {
void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException; void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException;
List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException; List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException;
} }

735
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java
View File

@ -1,18 +1,12 @@
package eu.eudat.service.reference; package eu.eudat.service.reference;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission; import eu.eudat.authorization.Permission;
import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive; import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType; import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.commons.types.reference.DefinitionEntity; import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity; import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.commons.types.referencetype.*; import eu.eudat.commons.types.referencetype.*;
@ -20,17 +14,18 @@ import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceEntity; import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.Reference; import eu.eudat.model.Reference;
import eu.eudat.model.ReferenceType;
import eu.eudat.model.builder.ReferenceBuilder; import eu.eudat.model.builder.ReferenceBuilder;
import eu.eudat.model.deleter.ReferenceDeleter; import eu.eudat.model.deleter.ReferenceDeleter;
import eu.eudat.model.persist.ReferencePersist; import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.model.persist.referencedefinition.DefinitionPersist; import eu.eudat.model.persist.referencedefinition.DefinitionPersist;
import eu.eudat.model.persist.referencedefinition.FieldPersist; import eu.eudat.model.persist.referencedefinition.FieldPersist;
import eu.eudat.query.ReferenceQuery; import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.ReferenceTypeQuery;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup; import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import eu.eudat.service.remotefetcher.RemoteFetcherService;
import eu.eudat.service.remotefetcher.models.ExternalRefernceResult; import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory; import gr.cite.tools.data.deleter.DeleterFactory;
@ -45,28 +40,17 @@ import gr.cite.tools.logging.LoggerService;
import gr.cite.tools.logging.MapLogEntry; import gr.cite.tools.logging.MapLogEntry;
import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.JAXBException;
import net.minidev.json.JSONArray;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.context.MessageSource; import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import javax.management.InvalidApplicationException; import javax.management.InvalidApplicationException;
import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerException;
import java.time.Instant; import java.time.Instant;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
@Service @Service
public class ReferenceServiceImpl implements ReferenceService { public class ReferenceServiceImpl implements ReferenceService {
@ -81,17 +65,16 @@ public class ReferenceServiceImpl implements ReferenceService {
private final QueryFactory queryFactory; private final QueryFactory queryFactory;
private final XmlHandlingService xmlHandlingService; private final XmlHandlingService xmlHandlingService;
private final WebClient client; public final RemoteFetcherService remoteFetcherService;
public ReferenceServiceImpl( public ReferenceServiceImpl(
EntityManager entityManager, EntityManager entityManager,
AuthorizationService authorizationService, AuthorizationService authorizationService,
DeleterFactory deleterFactory, DeleterFactory deleterFactory,
BuilderFactory builderFactory, BuilderFactory builderFactory,
ConventionService conventionService, ConventionService conventionService,
MessageSource messageSource, MessageSource messageSource,
QueryFactory queryFactory, QueryFactory queryFactory,
XmlHandlingService xmlHandlingService) { XmlHandlingService xmlHandlingService, RemoteFetcherService remoteFetcherService) {
this.entityManager = entityManager; this.entityManager = entityManager;
this.authorizationService = authorizationService; this.authorizationService = authorizationService;
this.deleterFactory = deleterFactory; this.deleterFactory = deleterFactory;
@ -100,11 +83,7 @@ public class ReferenceServiceImpl implements ReferenceService {
this.messageSource = messageSource; this.messageSource = messageSource;
this.queryFactory = queryFactory; this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService; this.xmlHandlingService = xmlHandlingService;
this.client = WebClient.builder().codecs(clientCodecConfigurer -> { this.remoteFetcherService = remoteFetcherService;
clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
}
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
} }
@Override @Override
@ -178,380 +157,340 @@ public class ReferenceServiceImpl implements ReferenceService {
this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id)); this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id));
} }
// public List<FetcherReference> searchReference(ReferenceType externalType, String query, String type) throws HugeResultSet, MyNotFoundException, InvalidApplicationException { @Override
// ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); public List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException {
// int initialOffset = 0;
// List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(externalType, externalUrlCriteria, type); if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
// initialOffset = lookup.getPage().getOffset();
// List<FetcherReference> list = this.fetchFromDb(externalType, query, type, remoteRepos); lookup.getPage().setOffset(0);
// }
// list.addAll(remoteRepos.stream().map(FetcherReference::fromRemoteModel).toList());
// list = list.stream().filter(x -> x.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
// list.sort(Comparator.comparing(FetcherReference::getName));
// return list;
// }
// @Override ReferenceTypeEntity data = this.entityManager.find(ReferenceTypeEntity.class, lookup.getTypeId());
// public List<Reference> searchReference(ReferenceSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException { if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{lookup.getTypeId(), ReferenceType.class.getSimpleName()}, LocaleContextHolder.getLocale()));
// int initialOffset = 0;
// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){ ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
// initialOffset = lookup.getPage().getOffset();
// lookup.getPage().setOffset(0);
// }
//
// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
//
// List<Map<String, String>> remoteRepos = remoteFetcherService.getReferences(lookup.getType(), externalReferenceCriteria, lookup.getKey());
//
// List<Reference> externalModels = this.builderFactory.builder(ReferenceSearchBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), remoteRepos);
// List<Reference> models = this.fetchFromDb(lookup);
//
// models.addAll(externalModels);
//
// if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).collect(Collectors.toList()); }
// models.sort(Comparator.comparing(Reference::getLabel));
//
// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
// models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList();
// }
//
// return models;
// }
private List<Reference> fetchFromDb(ReferenceSearchLookup lookup){ ExternalDataResult remoteRepos = this.getReferenceData(data, externalReferenceCriteria, lookup.getKey());
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal);
List<Reference> externalModels = new ArrayList<>();
if (remoteRepos != null && this.conventionService.isListNullOrEmpty(remoteRepos.getResults())){
List<ReferenceEntity> referenceEntities = new ArrayList<>();
for (Map<String, String> result : remoteRepos.getResults()){
if (result == null || result.isEmpty()) continue;;
ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data, remoteRepos);
referenceEntities.add(referenceEntity);
}
externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities);
}
List<Reference> models = this.fetchReferenceFromDb(lookup);
for (Reference externalReference : externalModels){
if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null &&x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference);
}
if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).toList(); }
models.sort(Comparator.comparing(Reference::getLabel));
if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList();
}
return models;
}
@NotNull
private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) {
ReferenceEntity referenceEntity = new ReferenceEntity(); //TODO new reference logic
referenceEntity.setTypeId(data.getId());
referenceEntity.setIsActive(IsActive.Active);
referenceEntity.setReference(result.getOrDefault("key", null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("pid", null));
referenceEntity.setSource(result.getOrDefault("tag", null));
referenceEntity.setAbbreviation(result.getOrDefault("abbreviation", null));
referenceEntity.setDescription(result.getOrDefault("description", null));
referenceEntity.setLabel(result.getOrDefault("name", null));
referenceEntity.setSourceType(ReferenceSourceType.External);
DefinitionEntity definitionEntity = new DefinitionEntity();
definitionEntity.setFields(new ArrayList<>());
for (Map.Entry<String, String> resultValue : result.entrySet()){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode(resultValue.getKey());
fieldEntity.setValue(resultValue.getValue());
fieldEntity.setDataType(ReferenceFieldDataType.Text);
definitionEntity.getFields().add(fieldEntity);
}
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
return referenceEntity;
}
private List<Reference> fetchReferenceFromDb(ReferenceSearchLookup lookup){
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId());
List<ReferenceEntity> data = query.collectAs(lookup.getProject()); List<ReferenceEntity> data = query.collectAs(lookup.getProject());
return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data); return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data);
} }
// private List<FetcherReference> fetchFromDb(ReferenceType externalType, String query, String type, List<Map<String, String>> remoteRepos) throws InvalidApplicationException { private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) {
// List<FetcherReference> list = new LinkedList<>(); ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition());
// switch (externalType) { if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult();
// case DataRepositories:
// case PubRepositories: ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).toList(), externalReferenceCriteria, key, null);
// case Journals: { for (Map<String, String> result: results.getResults()) {
// DataRepositoryCriteria criteria = new DataRepositoryCriteria(); result.put("referenceType", referenceType.getName());
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<DataRepository> dataRepositoryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao().getWithCriteria(criteria)).toList();
// list = dataRepositoryList.stream().map(item -> new FetcherReference().fromDataRepository(item)).collect(Collectors.toList());
// }
// }
// case Registries: {
// RegistryCriteria criteria = new RegistryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<Registry> registryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao().getWithCriteria(criteria)).toList();
// list = registryList.stream().map(item -> new FetcherReference().fromRegistry(item)).collect(Collectors.toList());
// }
// }
// case Services:
// {
// ServiceCriteria criteria = new ServiceCriteria();
//
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
//
// if (type.equals("")) {
// List<Service> serviceList = (this.apiContext.getOperationsContext().getDatabaseRepository().getServiceDao().getWithCriteria(criteria)).toList();
// list = serviceList.stream().map(item -> new FetcherReference().fromService(item)).collect(Collectors.toList());
// }
// }
// case Datasets:{
//
// ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build();
//
// criteria.setCreationUserId(this.userScope.getUserId());
// QueryableList<ExternalDataset> items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria);
//
// list = items.select(item -> new FetcherReference().fromDataset(item));
// }
// case Taxonomies:
// case Publications:
// case Licenses:
// break;
// }
//
// return list;
// }
@Override
public List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException {
ReferenceTypeQuery query = this.queryFactory.query(ReferenceTypeQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(lookup.getReferenceTypeId());
List<ReferenceTypeEntity> datas = query.collectAs(lookup.getProject());
if (datas.size() != 1 ){
return null;
} }
ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, datas.get(0).getDefinition());
List<Map<String, String>> remoteRepos = this.getAll(definition.getSources(), lookup);
return null;
}
private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
List<Map<String, String>> results = new LinkedList<>();
if (sources == null || sources.isEmpty()) {
return results;
}
sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
apiSources.forEach(source -> {
try {
String auth = null;
if (source.getAuth()!= null) {
//auth = this.getAuthentication(source.getAuth());
}
results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
});
List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
staticSources.forEach(source -> {
Map<String, String> map = new HashMap<>();
source.getOptions().forEach(option -> {
map.put(option.getCode(), option.getValue());
map.put("tag", source.getLabel());
map.put("key", source.getKey());
});
results.add(map);
});
return results; return results;
} }
private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) { // private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name()); // List<Map<String, String>> results = new LinkedList<>();
Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl()) //
.contentType(MediaType.APPLICATION_JSON) // if (sources == null || sources.isEmpty()) {
.bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) // return results;
.exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() { // }
})).block(); //
// sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
//
// List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
// apiSources.forEach(source -> {
// try {
// String auth = null;
// if (source.getAuth()!= null) {
// //auth = this.getAuthentication(source.getAuth());
// }
// results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// });
// List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
// staticSources.forEach(source -> {
// Map<String, String> map = new HashMap<>();
// source.getOptions().forEach(option -> {
// map.put(option.getCode(), option.getValue());
// map.put("tag", source.getLabel());
// map.put("key", source.getKey());
// });
// results.add(map);
// });
//
// return results;
// }
// private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
// Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
// .contentType(MediaType.APPLICATION_JSON)
// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
// })).block();
//
//
//
// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
// }
// private String parseBodyString(String bodyString) {
// String finalBodyString = bodyString;
// if (bodyString.contains("{env:")) {
// int index = bodyString.indexOf("{env: ");
// while (index >= 0) {
// int endIndex = bodyString.indexOf("}", index + 6);
// String envName = bodyString.substring(index + 6, endIndex);
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// index = bodyString.indexOf("{env: ", index + 6);
// }
// }
// return finalBodyString;
// }
// private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
// Set<Integer> pages = new HashSet<>();
//
// String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
// String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
//
// ExternalDataResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
// if(externalRefernceResult != null) {
// if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
// .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
// .collect(Collectors.toList()));
// }
// if (fetchStrategy == FetchStrategy.FIRST)
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
//
// //Long maxResults = configLoader.getExternalUrls().getMaxresults();
// Long maxResults = Long.valueOf(1000);
// if (externalRefernceResult.getResults().size() > maxResults)
// throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getResults().size() + " results... Please submit a more detailed search query");
//
// Optional<ExternalDataResult> optionalResults = pages.parallelStream()
// .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
// .filter(Objects::nonNull)
// .reduce((result1, result2) -> {
// result1.getResults().addAll(result2.getResults());
// return result1;
// });
// ExternalDataResult remainingExternalDataResult = optionalResults.orElseGet(ExternalDataResult::new);
// remainingExternalDataResult.getResults().addAll(externalRefernceResult.getResults());
//
// return remainingExternalDataResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
// }
// else {
// return new LinkedList<>();
// }
// }
// private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
// String completedPath = urlPath;
//
// if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
// if (lookup.getLike() != null) {
// completedPath = completedPath.replace("{like}", lookup.getLike());
// } else {
// completedPath = completedPath.replace("{like}", "*");
// }
// }
//
// if (urlPath.contains("{like}")){
// if (lookup.getLike() != null) {
// completedPath = completedPath.replace("{like}", lookup.getLike());
// } else {
// completedPath = completedPath.replace("{like}", "");
// }
// }
//
// if (urlPath.contains("{page}")) {
// if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
// completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
// } else if (firstPage != null) {
// completedPath = completedPath.replace("{page}", firstPage);
// } else {
// completedPath = completedPath.replace("{page}", "1");
// }
// }
//
// if (urlPath.contains("{pageSize}")){
// if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
// completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
// } else {
// completedPath = completedPath.replace("{pageSize}", "100");
// }
// }
//
//
// return completedPath;
// }
// protected ExternalDataResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
//
// try {
// ResponseEntity<String> response;
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
//
// response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
// if (contentType != null && !contentType.isEmpty()) {
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
// httpHeaders.setContentType(MediaType.valueOf(contentType));
// }
// if (auth != null) {
// httpHeaders.set("Authorization", auth);
// }
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
// if (response.getStatusCode() == HttpStatus.OK) { // success
// ExternalDataResult externalDataResult = new ExternalDataResult();
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
// externalDataResult = this.getFromJson(jsonContext, resultsEntity);
// }
//
// return externalDataResult;
// }
// } catch (Exception exception) {
// logger.error(exception.getMessage(), exception);
// }
//
// return null;
// }
// public static ExternalDataResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath()); // return new ExternalDataResult(parseData(jsonContext, resultsEntity));
} // }
private String parseBodyString(String bodyString) {
String finalBodyString = bodyString;
if (bodyString.contains("{env:")) {
int index = bodyString.indexOf("{env: ");
while (index >= 0) {
int endIndex = bodyString.indexOf("}", index + 6);
String envName = bodyString.substring(index + 6, endIndex);
finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
index = bodyString.indexOf("{env: ", index + 6);
}
}
return finalBodyString;
}
private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
Set<Integer> pages = new HashSet<>();
String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
if(externalRefernceResult != null) {
if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
if (fetchStrategy == FetchStrategy.FIRST)
return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set
for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++)
pages.add(i);
//Long maxResults = configLoader.getExternalUrls().getMaxresults();
Long maxResults = Long.valueOf(1000);
if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults))
throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query");
Optional<ExternalRefernceResult> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
.filter(Objects::nonNull)
.reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults());
return result1;
});
ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults());
return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
}
else {
return new LinkedList<>();
}
}
private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
String completedPath = urlPath;
if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
if (lookup.getLike() != null) {
completedPath = completedPath.replace("{like}", lookup.getLike());
} else {
completedPath = completedPath.replace("{like}", "*");
}
}
if (urlPath.contains("{like}")){
if (lookup.getLike() != null) {
completedPath = completedPath.replace("{like}", lookup.getLike());
} else {
completedPath = completedPath.replace("{like}", "");
}
}
if (urlPath.contains("{page}")) {
if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
} else if (firstPage != null) {
completedPath = completedPath.replace("{page}", firstPage);
} else {
completedPath = completedPath.replace("{page}", "1");
}
}
if (urlPath.contains("{pageSize}")){
if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
} else {
completedPath = completedPath.replace("{pageSize}", "100");
}
}
return completedPath; // private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
} // List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
// List<Map<String, String>> parsedData = new ArrayList<>();
protected ExternalRefernceResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) { //
// for (Map<String, String> stringObjectMap: rawData){
try { // Map<String, String> map = new HashMap<>();
ResponseEntity<String> response; // for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
JsonNode jsonBody = new ObjectMapper().readTree(requestBody); // String pathValue = field.getResponsePath();
// if (!pathValue.contains(".")){
response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> { // if (stringObjectMap.containsKey(pathValue)) {
if (contentType != null && !contentType.isEmpty()) { // //map.put(field.getCode(), stringObjectMap.get(pathValue));
httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); // map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
httpHeaders.setContentType(MediaType.valueOf(contentType)); // }
} // }else {
if (auth != null) { // if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
httpHeaders.set("Authorization", auth); // String value = null;
} // Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
}).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); // if (fieldObj != null){
if (response.getStatusCode() == HttpStatus.OK) { // success // if (fieldObj instanceof Map){
ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(); // Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
if (response.getHeaders().get("Content-Type").get(0).contains("json")) { // if(o instanceof String){
DocumentContext jsonContext = JsonPath.parse(response.getBody()); // value = (String)o;
externalRefernceResult = this.getFromJson(jsonContext, resultsEntity); // }
} // else if(o instanceof Integer){
// value = String.valueOf(o);
// }
if (externalRefernceResult.getPagination().size() == 0) { // } else if (fieldObj instanceof List) {
externalRefernceResult.getPagination().put("pages", 1); // Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size()); // if(o instanceof String){
} // value = (String)o;
return externalRefernceResult; // }
} // else if(o instanceof Integer){
} catch (Exception exception) { // value = String.valueOf(o);
logger.error(exception.getMessage(), exception); // }
} // }
// }
return null; // if (value != null){
} // map.put(field.getCode(), value);
// }
// }
public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { // }
return new ExternalRefernceResult(parseData(jsonContext, resultsEntity), // }
new HashMap<>(1, 1)); // parsedData.add(map);
} // }
// return parsedData;
// }
private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { //
List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath()); //
List<Map<String, String>> parsedData = new ArrayList<>(); // private static String normalizeValue(Object value) {
// if (value instanceof JSONArray) {
for (Map<String, String> stringObjectMap: rawData){ // JSONArray jarr = (JSONArray) value;
Map<String, String> map = new HashMap<>(); // if (jarr.get(0) instanceof String) {
for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){ // return jarr.get(0).toString();
String pathValue = field.getResponsePath(); // } else {
if (!pathValue.contains(".")){ // for (Object o : jarr) {
if (stringObjectMap.containsKey(pathValue)) { // if ((o instanceof Map) && ((Map) o).containsKey("content")) {
//map.put(field.getCode(), stringObjectMap.get(pathValue)); // try {
map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue))); // return ((Map<String, String>) o).get("content");
} // }
}else { // catch (ClassCastException e){
if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){ // if(((Map<?, ?>) o).get("content") instanceof Integer) {
String value = null; // return String.valueOf(((Map<?, ?>) o).get("content"));
Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]); // }
if (fieldObj != null){ // return null;
if (fieldObj instanceof Map){ // }
Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]); // }
if(o instanceof String){ // }
value = (String)o; // }
} // } else if (value instanceof Map) {
else if(o instanceof Integer){ // String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
value = String.valueOf(o); // return ((Map<String, String>)value).get(key);
} // }
} else if (fieldObj instanceof List) { // return value != null ? value.toString() : null;
Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]); // }
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
}
}
if (value != null){
map.put(field.getCode(), value);
}
}
}
}
parsedData.add(map);
}
return parsedData;
}
private static String normalizeValue(Object value) {
if (value instanceof JSONArray) {
JSONArray jarr = (JSONArray) value;
if (jarr.get(0) instanceof String) {
return jarr.get(0).toString();
} else {
for (Object o : jarr) {
if ((o instanceof Map) && ((Map) o).containsKey("content")) {
try {
return ((Map<String, String>) o).get("content");
}
catch (ClassCastException e){
if(((Map<?, ?>) o).get("content") instanceof Integer) {
return String.valueOf(((Map<?, ?>) o).get("content"));
}
return null;
}
}
}
}
} else if (value instanceof Map) {
String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
return ((Map<String, String>)value).get(key);
}
return value != null ? value.toString() : null;
}
} }

1
dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java
View File

@ -18,7 +18,6 @@ import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl;
import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory; import gr.cite.tools.data.deleter.DeleterFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException; import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException; import gr.cite.tools.exception.MyNotFoundException;

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java
View File

@ -1,21 +1,14 @@
//package eu.eudat.service.remotefetcher; package eu.eudat.service.remotefetcher;
//
//import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.data.ReferenceTypeEntity;
//import eu.eudat.service.remotefetcher.config.entities.GenericUrls; import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import gr.cite.tools.exception.MyNotFoundException; import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
// import eu.eudat.service.remotefetcher.models.ExternalDataResult;
//import java.util.List;
//import java.util.Map; import java.util.List;
//
//public interface RemoteFetcherService { public interface RemoteFetcherService {
// List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException; ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy);
// Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
// GenericUrls getExternalUrls(ReferenceType referenceType); }
//
// Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException;
//
// List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
//
// List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
//}

917
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java
View File

@ -1,533 +1,388 @@
//package eu.eudat.service.remotefetcher; package eu.eudat.service.remotefetcher;
//
//import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
//import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
//import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
//import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.DocumentContext;
//import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.JsonPath;
//import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.commons.XmlHandlingService;
//import eu.eudat.service.remotefetcher.config.AuthenticationConfiguration; import eu.eudat.commons.enums.ReferenceTypeSourceType;
//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration; import eu.eudat.commons.exceptions.HugeResultSetException;
//import eu.eudat.service.remotefetcher.config.QueryConfig; import eu.eudat.convention.ConventionService;
//import eu.eudat.service.remotefetcher.config.UrlConfiguration; import eu.eudat.service.remotefetcher.config.entities.*;
//import eu.eudat.service.remotefetcher.config.entities.GenericUrls; import eu.eudat.service.remotefetcher.models.ExternalDataResult;
//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
//import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.exception.MyNotFoundException; import net.minidev.json.JSONArray;
//import jakarta.xml.bind.JAXBContext; import org.slf4j.Logger;
//import jakarta.xml.bind.Unmarshaller; import org.slf4j.LoggerFactory;
//import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired;
//import org.slf4j.LoggerFactory; import org.springframework.core.ParameterizedTypeReference;
//import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.*;
//import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.client.reactive.ReactorClientHttpConnector;
//import org.springframework.http.*; import org.springframework.http.codec.json.Jackson2JsonDecoder;
//import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.stereotype.Service;
//import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.web.reactive.function.client.WebClient;
//import org.springframework.stereotype.Service; import reactor.netty.http.client.HttpClient;
//import org.springframework.web.client.RestTemplate;
//import org.springframework.web.reactive.function.client.WebClient; import java.io.File;
//import reactor.netty.http.client.HttpClient; import java.nio.file.Paths;
// import java.util.*;
//import java.io.File; import java.util.stream.Collectors;
//import java.io.StringReader;
//import java.lang.reflect.Method; @Service
//import java.nio.file.Paths; public class RemoteFetcherServiceImpl implements RemoteFetcherService {
//import java.util.*; private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
//import java.util.stream.Collectors;
// private WebClient webClient;
//@Service private final ExternalUrlConfigProvider externalUrlConfigProvider;
//public class RemoteFetcherServiceImpl implements RemoteFetcherService { private final ConventionService conventionService;
// private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class); private final XmlHandlingService xmlHandlingService;
// @Autowired
// private WebClient webClient; public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) {
// private final ExternalUrlConfigProvider externalUrlConfigProvider; this.externalUrlConfigProvider = externalUrlConfigProvider;
// @Autowired this.conventionService = conventionService;
// public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider) { this.xmlHandlingService = xmlHandlingService;
// this.externalUrlConfigProvider = externalUrlConfigProvider; }
// }
// private WebClient getWebClient(){
// private WebClient getWebClient(){ if (this.webClient == null) {
// if (this.webClient == null) { this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> {
// this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> { clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
// clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON)); clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
// clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here??? }
// } ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
// ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build(); }
// } return webClient;
// return webClient; }
// }
//
// @Override
// public List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException { @Override
// FetchStrategy fetchStrategy = null; public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) {
// List<SourceBaseConfiguration> apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList();
//// GenericUrls exGenericUrls = this.getExternalUrls(referenceType); if (!this.conventionService.isNullOrEmpty(key)){
// List<UrlConfiguration> urlConfigs = key != null && !key.isEmpty() ? exGenericUrls.getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList();
// : exGenericUrls.getUrls(); }
// if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult();
// List<Map<String, String>> results = getAll(urlConfigs, fetchStrategy, externalReferenceCriteria);
// for (Map<String, String> result: results) { apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal));
// result.put("referenceType", referenceType.name());
// } return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria);
// return results; }
// }
// @Override
// @Override public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
// public GenericUrls getExternalUrls(ReferenceType referenceType) { return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size();
// return switch (referenceType) { }
// case Taxonomies -> this.externalUrlConfigProvider.getExternalUrls().getTaxonomies();
// case Licenses -> this.externalUrlConfigProvider.getExternalUrls().getLicenses(); private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
// case Publications -> this.externalUrlConfigProvider.getExternalUrls().getPublications();
// case Journals -> this.externalUrlConfigProvider.getExternalUrls().getJournals(); ExternalDataResult results = new ExternalDataResult();
// case PubRepositories -> this.externalUrlConfigProvider.getExternalUrls().getPubRepositories();
// case DataRepositories -> this.externalUrlConfigProvider.getExternalUrls().getRepositories(); if (this.conventionService.isListNullOrEmpty(sources)) return new ExternalDataResult();
// case Registries -> this.externalUrlConfigProvider.getExternalUrls().getRegistries();
// case Services -> this.externalUrlConfigProvider.getExternalUrls().getServices(); for (SourceBaseConfiguration source : sources) {
// case Grants -> this.externalUrlConfigProvider.getExternalUrls().getGrants(); if (source.getType() == null || source.getType().equals(ReferenceTypeSourceType.API)) {
// case Organizations -> this.externalUrlConfigProvider.getExternalUrls().getOrganisations(); try {
// case Datasets -> this.externalUrlConfigProvider.getExternalUrls().getDatasets(); SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource = (SourceExternalApiConfiguration)source;
// case Funder -> this.externalUrlConfigProvider.getExternalUrls().getFunders(); this.applyFunderToQuery(apiSource, externalReferenceCriteria);
// case Project -> this.externalUrlConfigProvider.getExternalUrls().getProjects();
// case Researcher -> this.externalUrlConfigProvider.getExternalUrls().getResearchers(); String auth = null;
// default -> throw new IllegalArgumentException("Type not found" + referenceType); if (apiSource.getAuth() != null) {
// }; auth = this.buildAuthentication(apiSource.getAuth());
// } }
// results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth));
// @Override } catch (Exception e) {
// public Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException { logger.error(e.getLocalizedMessage(), e);
// List<UrlConfiguration> urlConfigs = }
// key != null && !key.isEmpty() ? this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) } else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) {
// : this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls(); SourceStaticOptionConfiguration<StaticOption> staticSource = (SourceStaticOptionConfiguration)source;
// FetchStrategy fetchStrategy = this.externalUrlConfigProvider.getExternalUrls().getValidations().getFetchMode(); results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO:
// List<Map<String, String>> data = this.getAll(urlConfigs, fetchStrategy, externalReferenceCriteria); }
// return data.size(); }
// } return results;
// }
// @Override
// public List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) { private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) {
// List<UrlConfiguration> urlConfigurations = genericUrls.getUrls(); HttpMethod method;
// FetchStrategy fetchStrategy = genericUrls.getFetchMode(); switch (authenticationConfiguration.getAuthMethod()) {
// return getAll(urlConfigurations, fetchStrategy, externalReferenceCriteria); case GET -> method = HttpMethod.GET;
// } case POST -> method =HttpMethod.POST;
// default -> throw new MyApplicationException("unrecognized type " + authenticationConfiguration.getAuthMethod());
// @Override }
// public List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) {
// List<UrlConfiguration> urlConfigurations = genericUrls.getUrls(); Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
// return getAllWithData(urlConfigurations, externalReferenceCriteria); .contentType(MediaType.APPLICATION_JSON)
// } .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
// private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) { })).block();
// if (response == null) throw new MyApplicationException("Authentication " + authenticationConfiguration.getAuthUrl() + " failed");
// List<Map<String, String>> results = new LinkedList<>();
// return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null);
// if (urlConfigs == null || urlConfigs.isEmpty()) { }
// return results;
// }
//// throw new MyNotFoundException("No Repository urls found in configuration"); private void applyFunderToQuery(SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) {
// //TODO new reference logic
// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal)); // if (apiSource.getFunderQuery() != null) {
// for (UrlConfiguration urlConfiguration : urlConfigs) { // if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) {
// applyFunderQuery(urlConfiguration, externalReferenceCriteria); // apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery()));
// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) { // }
// try { // else {
// String auth = null; // apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", ""));
// if (urlConfiguration.getAuth() != null) {
// auth = this.getAuthentication(urlConfiguration.getAuth());
// }
// results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalReferenceCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType(), urlConfiguration.getQueries(), auth));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// } else if (urlConfiguration.getType() != null && urlConfiguration.getType().equals("Internal")) {
// results.addAll(getAllResultsFromMockUpJson(urlConfiguration.getUrl(), externalReferenceCriteria.getLike()));
// } // }
// } // }
// /* for (UrlConfiguration urlConfig : urlConfigs) { }
// ifFunderQueryExist(urlConfig, externalUrlCriteria);
// if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { private String replaceLookupFieldQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
// results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType())); String finalQuery = "";
// } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { if (this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike())) return finalQuery;
// results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike())); QueryConfig queryConfig = queryConfigs.stream().filter(x -> !this.conventionService.isNullOrEmpty(x.getCondition()) && externalReferenceCriteria.getLike().matches(x.getCondition()))
// } .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
// }*/ if (queryConfig != null) {
// return results; if (queryConfig.getSeparator() != null) {
// } String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
// finalQuery = queryConfig.getValue();
// private String getAuthentication(AuthenticationConfiguration authenticationConfiguration) { for (int i = 0; i < likes.length; i++) {
// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod()); finalQuery = finalQuery.replaceAll("\\{like" + (i + 1) + "}", likes[i]);
// Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl()) }
// .contentType(MediaType.APPLICATION_JSON) } else {
// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike());
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() { }
// })).block();
// }
// return finalQuery;
// }
// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
// } protected String replaceLookupFields(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
// if (!this.conventionService.isNullOrEmpty(path)) return path;
// private List<Map<String, Object>> getAllWithData(List<UrlConfiguration> urlConfigs, ExternalReferenceCriteria externalReferenceCriteria) { String completedPath = path;
//
// List<Map<String, Object>> results = new LinkedList<>(); if (externalReferenceCriteria.getLike() != null) {
// if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().isEmpty()) {
// if (urlConfigs == null || urlConfigs.isEmpty()) { completedPath = completedPath.replaceAll("\\{like}", "*");
// return results; completedPath = completedPath.replaceAll("\\{query}", "*");
// } } else {
// if (completedPath.contains("{query}")) {
// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal)); completedPath = completedPath.replaceAll("\\{query}", this.replaceLookupFieldQuery(externalReferenceCriteria, queries));
// urlConfigs.forEach(urlConfiguration -> { } else {
// applyFunderQuery(urlConfiguration, externalReferenceCriteria); completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike());
// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) { }
// try { }
// results.addAll(getAllResultsFromUrlWithData(urlConfiguration.getUrl(), urlConfiguration.getData(), externalReferenceCriteria, urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getQueries())); } else {
// } catch (Exception e) { completedPath = completedPath.replace("{like}", "");
// logger.error(e.getLocalizedMessage(), e); }
// } if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getFunderId())) {
// } String[] founderParts = externalReferenceCriteria.getFunderId().split(":");
// }); String funderPrefix = founderParts.length > 0 ? founderParts[0] : "";
// return results; String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", "");
// if (!funderId.isEmpty() && funderId.toCharArray()[0] == ':') {
// } funderId = externalReferenceCriteria.getFunderId();
// }
// private void applyFunderQuery(UrlConfiguration urlConfiguration, ExternalReferenceCriteria externalReferenceCriteria) { completedPath = completedPath.replace("{funderId}", funderId);
// if (urlConfiguration.getFunderQuery() != null) { } else if(completedPath.contains("{funderId}")){
// if (externalReferenceCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) { logger.warn("FunderId is null.");
// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery())); completedPath = completedPath.replace("{funderId}", " ");
// } }
// else {
// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", "")); if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPage())) completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
// } else if (!this.conventionService.isNullOrEmpty(firstPage)) completedPath = completedPath.replace("{page}", firstPage);
// } else completedPath = completedPath.replace("{page}", "1");
// }
// completedPath = completedPath.replace("{pageSize}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPageSize()) ? externalReferenceCriteria.getPageSize() : "60");
// private String calculateQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) { completedPath = completedPath.replace("{host}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getHost()) ? externalReferenceCriteria.getHost() : "");
// String finalQuery = ""; completedPath = completedPath.replace("{path}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPath()) ? externalReferenceCriteria.getPath() : "");
// QueryConfig queryConfig = queryConfigs.stream().filter(queryConfigl -> externalReferenceCriteria.getLike().matches(queryConfigl.getCondition()))
// .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
// if (queryConfig != null) { return completedPath;
// if (queryConfig.getSeparator() != null) { }
// String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
// finalQuery = queryConfig.getValue(); private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
// for (int i = 0; i < likes.length; i++) { String replacedPath = replaceLookupFields(apiSource.getUrl(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
// finalQuery = finalQuery.replaceAll("\\{like" + (i+1) + "}", likes[i]); String replacedBody = replaceLookupFields(apiSource.getRequestBody(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
// }
// } else { ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
// finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike()); if(externalDataResult != null) {
// } if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
// externalDataResult.setResults(externalDataResult.getResults().stream()
// } .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
// return finalQuery; .collect(Collectors.toList()));
// } }
// externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList());
// protected String replaceCriteriaOnUrl(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) { if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
// String completedPath = path;
// if (externalReferenceCriteria.getLike() != null) { Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
// if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().equals("")) { if (externalDataResult.getResults().size() > maxResults)
// completedPath = completedPath.replaceAll("\\{like}", "*"); throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query");
// completedPath = completedPath.replaceAll("\\{query}", "*");
// } else { return externalDataResult;
// if (completedPath.contains("{query}")) { }
// completedPath = completedPath.replaceAll("\\{query}", this.calculateQuery(externalReferenceCriteria, queries)); else {
// } else { return new ExternalDataResult();
// completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike()); }
// } }
// }
// } else { protected ExternalDataResult getExternalDataResults(String urlString, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, String requestBody, String auth) {
// completedPath = completedPath.replace("{like}", "");
// } try {
// if (externalReferenceCriteria.getFunderId() != null) { JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
// String funderPrefix = externalReferenceCriteria.getFunderId().split(":")[0]; HttpMethod method;
// String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", ""); switch (apiSource.getHttpMethod()) {
// if (funderId.toCharArray()[0] == ':') { case GET -> method = HttpMethod.GET;
// funderId = externalReferenceCriteria.getFunderId(); case POST -> method =HttpMethod.POST;
// } default -> throw new MyApplicationException("unrecognized type " + apiSource.getHttpMethod());
// /* }
// try { funderId = URLEncoder.encode(funderId, "UTF-8"); } catch
// (UnsupportedEncodingException e) { logger.error(e.getMessage(), e); } ResponseEntity<String> response = this.getWebClient().method(method).uri(urlString).headers(httpHeaders -> {
// */ if (this.conventionService.isNullOrEmpty(apiSource.getContentType())) {
// completedPath = completedPath.replace("{funderId}", funderId); httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(apiSource.getContentType())));
// } httpHeaders.setContentType(MediaType.valueOf(apiSource.getContentType()));
// else if(completedPath.contains("{funderId}")){ }
// logger.warn("FunderId is null."); if (auth != null) {
// completedPath = completedPath.replace("{funderId}", " "); httpHeaders.set("Authorization", auth);
// } }
// if (externalReferenceCriteria.getPage() != null) { }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
// completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage()); if (response == null || !response.getStatusCode().isSameCodeAs(HttpStatus.OK) || !response.hasBody() || response.getBody() == null) return null;
// } else {
// if (firstPage != null) { //do here all the parsing
// completedPath = completedPath.replace("{page}", firstPage); List<String> responseContentTypeHeader = response.getHeaders().getOrDefault("Content-Type", null);
// } else { String responseContentType = !this.conventionService.isListNullOrEmpty(responseContentTypeHeader) && responseContentTypeHeader.getFirst() != null ? responseContentTypeHeader.getFirst() : "";
// completedPath = completedPath.replace("{page}", "1");
// } if (responseContentType.contains("json") ) {
// } DocumentContext jsonContext = JsonPath.parse(response.getBody());
// if (externalReferenceCriteria.getPageSize() != null) { return this.parseData(jsonContext, apiSource.getResults());
// completedPath = completedPath.replace("{pageSize}", externalReferenceCriteria.getPageSize()); } else {
// } else { throw new MyApplicationException("Unsupported response type" + responseContentType);
// completedPath = completedPath.replace("{pageSize}", "60"); }
// }
// if (externalReferenceCriteria.getHost() != null) { } catch (Exception exception) {
// completedPath = completedPath.replace("{host}", externalReferenceCriteria.getHost()); logger.error(exception.getMessage(), exception);
// } else { }
// completedPath = completedPath.replace("{host}", "");
// } return null;
// if (externalReferenceCriteria.getPath() != null) { }
// completedPath = completedPath.replace("{path}", externalReferenceCriteria.getPath());
// } else { private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration<ResultFieldsMappingConfiguration> resultsConfigurationEntity) {
// completedPath = completedPath.replace("{path}", ""); ExternalDataResult result = new ExternalDataResult();
// } if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult();
// return completedPath; List<Map<String, Object>> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath());
// }
// result.setRawData(rawData);
// private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalReferenceCriteria externalReferenceCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType, List<QueryConfig> queries, String auth) throws Exception {
// Set<Integer> pages = new HashSet<>(); if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult();
//
// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries); List<Map<String, String>> parsedData = new ArrayList<>();
// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries); for (Map<String, Object> stringObjectMap: rawData){
// Map<String, String> map = new HashMap<>();
// ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth); for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){
// if(externalRefernceResult != null) { String pathValue = field.getResponsePath();
// if (filterType != null && filterType.equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) { if (!pathValue.contains(".")){
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream() if (stringObjectMap.containsKey(pathValue)) {
// .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase())) map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
// .collect(Collectors.toList())); }
// } }else {
// if (fetchStrategy == FetchStrategy.FIRST) if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); String value = null;
// Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
// if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set if (fieldObj != null){
// for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++) if (fieldObj instanceof Map){
// pages.add(i); Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
// if(o instanceof String){
// Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults(); value = (String)o;
// if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults)) }
// throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query"); else if(o instanceof Integer){
// value = String.valueOf(o);
// Optional<ExternalRefernceResult> optionalResults = pages.parallelStream() }
// .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth)) } else if (fieldObj instanceof List) {
// .filter(Objects::nonNull) Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
// .reduce((result1, result2) -> { if(o instanceof String){
// result1.getResults().addAll(result2.getResults()); value = (String)o;
// return result1; }
// }); else if(o instanceof Integer){
// ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new); value = String.valueOf(o);
// remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults()); }
// }
// return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); }
// } if (value != null){
// else { map.put(field.getCode(), value);
// return new LinkedList<>(); }
// } }
// } }
// }
// private List<Map<String, Object>> getAllResultsFromUrlWithData(String path, final DataUrlConfiguration jsonDataPath, ExternalReferenceCriteria externalReferenceCriteria, String contentType, String firstPage, String requestBody, String requestType, List<QueryConfig> queries) { parsedData.add(map);
// }
// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries); result.setResults(parsedData);
// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries); return result;
// }
// try {
// RestTemplate restTemplate = new RestTemplate(); private static String normalizeValue(Object value) {
// HttpHeaders headers = new HttpHeaders(); if (value instanceof JSONArray jsonArray) {
// HttpEntity<JsonNode> entity;
// ResponseEntity<String> response; if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
// if (contentType != null && !contentType.isEmpty()) { return jsonArray.getFirst().toString();
// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); } else {
// headers.setContentType(MediaType.valueOf(contentType)); for (Object o : jsonArray) {
// } if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
// JsonNode jsonBody = new ObjectMapper().readTree(replacedBody); try {
// entity = new HttpEntity<>(jsonBody, headers); return ((Map<String, String>) o).get("content");
// }
// response = restTemplate.exchange(replacedPath, HttpMethod.valueOf(requestType), entity, String.class); catch (ClassCastException e){
// if (response.getStatusCode() == HttpStatus.OK) { if(((Map<?, ?>) o).get("content") instanceof Integer) {
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) { return String.valueOf(((Map<?, ?>) o).get("content"));
// DocumentContext jsonContext = JsonPath.parse(response.getBody()); }
// return jsonContext.read(jsonDataPath.getPath()); return null;
// } }
// } }
// } }
// catch (Exception exception) { }
// logger.error(exception.getMessage(), exception); } else if (value instanceof Map) {
// } String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
// return ((Map<String, String>)value).get(key);
// return new LinkedList<>(); }
// } return value != null ? value.toString() : null;
// }
//
// protected ExternalRefernceResult getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType, String auth) { private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) {
// List<Map<String, String>> internalResults;
// try { try {
// //RestTemplate restTemplate = new RestTemplate(new SimpleClientHttpRequestFactory()); String filePath = Paths.get(path).toUri().toURL().toString();
// //HttpHeaders headers = new HttpHeaders(); ObjectMapper mapper = new ObjectMapper();
// //HttpEntity<JsonNode> entity; internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
// ResponseEntity<String> response; return new ExternalDataResult(searchListMap(internalResults, query));
// /* } catch (Exception e) {
// * URL url = new URL(urlString.replaceAll(" ", "%20")); logger.error(e.getMessage(), e);
// * return new ExternalDataResult();
// * HttpURLConnection con = (HttpURLConnection) url.openConnection(); }
// * con.setRequestMethod("GET"); }
// */
// /* if (contentType != null && !contentType.isEmpty()) { private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); List<Map<String, String>> list = new LinkedList<>();
// headers.setContentType(MediaType.valueOf(contentType)); for (Map<String, String> map : internalResults)
// } {
// if (auth != null) { if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
// headers.set("Authorization", auth); list.add(map);
// }*/ }
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody); if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
//// entity = new HttpEntity<>(jsonBody, headers); list.add(map);
// }
// }
// response = this.getWebClient().method(HttpMethod.valueOf(requestType)).uri(urlString).headers(httpHeaders -> { return list;
// if (contentType != null && !contentType.isEmpty()) { }
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
// httpHeaders.setContentType(MediaType.valueOf(contentType)); private String parseBodyString(String bodyString) {
// } String finalBodyString = bodyString;
// if (auth != null) { if (bodyString.contains("{env:")) {
// httpHeaders.set("Authorization", auth); int index = bodyString.indexOf("{env: ");
// } while (index >= 0) {
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); int endIndex = bodyString.indexOf("}", index + 6);
// //response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class); String envName = bodyString.substring(index + 6, endIndex);
// if (response.getStatusCode() == HttpStatus.OK) { // success finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// //do here all the parsing index = bodyString.indexOf("{env: ", index + 6);
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(); }
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) { }
// DocumentContext jsonContext = JsonPath.parse(response.getBody()); return finalBodyString;
// }
// if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) {
// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType, auth); }
// } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) {
// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
// } else {
// externalRefernceResult = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath);
// }
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
// .collect(Collectors.toList()));
// }
// else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) {
// Class<?> aClass = Class.forName(jsonDataPath.getParseClass());
// JAXBContext jaxbContext = JAXBContext.newInstance(aClass);
// Unmarshaller unmarshaller = jaxbContext.createUnmarshaller();
// StringReader stringReader = new StringReader(response.getBody());
// Object data = unmarshaller.unmarshal(stringReader);
// Method reader = null;
// if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) {
// String camelCaseGetter = jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty() ? "get" + jsonDataPath.getParseField().substring(0, 1).toUpperCase() + jsonDataPath.getParseField().substring(1) : "";
// reader = aClass.getMethod(camelCaseGetter);
// }
// ObjectMapper objectMapper = new ObjectMapper();
// List<Map<String, String>> values = new ArrayList<>();
// int max = 1;
// if (reader != null) {
// Object invokedField = reader.invoke(data);
// if (invokedField instanceof Collection) {
// max = ((Collection) invokedField).size();
// }
// }
// for (int i = 0; i< max; i++) {
// Object value;
// if (reader != null) {
// Object invokedField = reader.invoke(data);
// if (invokedField instanceof Collection) {
// value = ((Collection) invokedField).toArray()[i];
// } else {
// value = invokedField;
// }
// } else {
// value = data;
// }
// Map<String, String> map = objectMapper.convertValue(value, Map.class);
// if (jsonDataPath.getMergedFields() != null && !jsonDataPath.getMergedFields().isEmpty() && jsonDataPath.getMergedFieldName() != null && !jsonDataPath.getMergedFieldName().isEmpty()) {
// Map<String, String> finalMap = new HashMap<>();
// for (Map.Entry<String, String> entry : map.entrySet()) {
// if (jsonDataPath.getMergedFields().contains(entry.getKey())) {
// if (!finalMap.containsKey(jsonDataPath.getMergedFieldName())) {
// finalMap.put(jsonDataPath.getMergedFieldName(), entry.getValue());
// } else {
// finalMap.put(jsonDataPath.getMergedFieldName(), finalMap.get(jsonDataPath.getMergedFieldName()) + " " + entry.getValue());
// }
// } else {
// finalMap.put(entry.getKey(), entry.getValue());
// }
// }
// values.add(finalMap);
// } else {
// values.add(map);
// }
// }
// externalRefernceResult = new ExternalRefernceResult(values, new HashMap<>(1, 1));
// }
//
// if (externalRefernceResult.getPagination().isEmpty()) {
// externalRefernceResult.getPagination().put("pages", 1);
// externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size());
// }
// return externalRefernceResult;
// }
// } catch (Exception exception) {
// logger.error(exception.getMessage(), exception);
// } //maybe print smth...
//
// return null;
// }
//
// private List<Map<String, String>> getAllResultsFromMockUpJson(String path, String query) {
// List<Map<String, String>> internalResults;
// try {
// String filePath = Paths.get(path).toUri().toURL().toString();
// ObjectMapper mapper = new ObjectMapper();
// internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
// return searchListMap(internalResults, query);
// } catch (Exception e) {
// logger.error(e.getMessage(), e);
// return new LinkedList<>();
// }
// }
//
// private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
// List<Map<String, String>> list = new LinkedList<>();
// for (Map<String, String> map : internalResults)
// {
// if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
// list.add(map);
// }
// if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
// list.add(map);
// }
// }
// return list;
// }
//
// private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) {
// if(dataUrlConfiguration.getFieldsUrlConfiguration().getPid() == null)
// return "pid";
// else
// return "originalId";
// }
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPid() != null && key.equals("pid")) return "pid";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPidTypeField() != null && key.equals("pidTypeField")) return "pidTypeField";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPath() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getPath().replace("'",""))) return "path";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getHost() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getHost().replace("'",""))) return "host";
// return null;
// }
//
// private String parseBodyString(String bodyString) {
// String finalBodyString = bodyString;
// if (bodyString.contains("{env:")) {
// int index = bodyString.indexOf("{env: ");
// while (index >= 0) {
// int endIndex = bodyString.indexOf("}", index + 6);
// String envName = bodyString.substring(index + 6, endIndex);
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// index = bodyString.indexOf("{env: ", index + 6);
// }
// }
// return finalBodyString;
// }
//
//}

152
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherUtils.java
View File

@ -1,152 +0,0 @@
//package eu.eudat.service.remotefetcher;
//
//import com.fasterxml.jackson.databind.ObjectMapper;
//import com.jayway.jsonpath.DocumentContext;
//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
//import net.minidev.json.JSONArray;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//
//import java.lang.reflect.InvocationTargetException;
//import java.lang.reflect.Method;
//import java.util.*;
//import java.util.stream.Collectors;
//
//public class RemoteFetcherUtils {
// private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class);
// private static final ObjectMapper mapper = new ObjectMapper();
//
// public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// return new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
// }
//
// public static ExternalRefernceResult getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcherServiceImpl remoteFetcherService, String requestBody, String requestType, String auth) {
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
//
// List<Map<String, String>> multiResults = externalRefernceResult.getResults().stream().map(result -> {
// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria();
// externalReferenceCriteria.setPath(result.get("path"));
// externalReferenceCriteria.setHost(result.get("host"));
// String replacedPath = remoteFetcherService.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalReferenceCriteria, jsonDataPath.getUrlConfiguration().getFirstpage(), jsonDataPath.getUrlConfiguration().getQueries());
// return remoteFetcherService.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType, auth);
// }).filter(Objects::nonNull).map(externalRefernceResult1 -> externalRefernceResult1.getResults().get(0)).collect(Collectors.toList());
// return new ExternalRefernceResult(multiResults, new HashMap<>(1, 1));
// }
//
// public static ExternalRefernceResult getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
// externalRefernceResult.getResults().stream().forEach(entry -> {
// String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
// entry.put("name", name);
// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
// });
// return externalRefernceResult;
// }
//
// private static List<Map<String, String>> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// List <Map<String, Object>> rawData = jsonContext.read(jsonDataPath.getPath());
// List<Map<String, String>> parsedData = new ArrayList<>();
// rawData.forEach(stringObjectMap -> {
// parsedData.add(new LinkedHashMap<>());
// Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> {
// String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1);
// Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0);
// try {
// String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration()));
// if (value != null) {
// if (field.getName().equals("pid") || field.getName().equals("pidTypeField")) {
// String pid = null;
// Object pidObj = stringObjectMap.get(value.split("\\.")[0]);
// if(pidObj != null){
// if(pidObj instanceof Map){
// Object o = ((Map<String, Object>) pidObj).get(value.split("\\.")[1]);
// if(o instanceof String){
// pid = (String)o;
// }
// else if(o instanceof Integer){
// pid = String.valueOf(o);
// }
// }
// else if(pidObj instanceof List){
// Object o = ((List<Map<String,?>>) pidObj).get(0).get(value.split("\\.")[1]);
// if(o instanceof String){
// pid = (String)o;
// }
// else if(o instanceof Integer){
// pid = String.valueOf(o);
// }
// }
// }
// if(pid != null) {
// if ((field.getName().equals("pid"))){
// parsedData.get(parsedData.size() - 1).put("pid", pid);
// }
// else{
// parsedData.get(parsedData.size() - 1).put("pidTypeField", pid);
// }
// }
// } else {
// value = value.replace("'", "");
// if (value.contains(".")) {
// String[] parts = value.split("\\.");
// Map<String, Object> tempMap = stringObjectMap;
// for (int i = 0; i < parts.length; i++) {
// if (tempMap.containsKey(parts[i])) {
// if (i + 1 < parts.length) {
// tempMap = (Map<String, Object>) tempMap.get(parts[i]);
// } else {
// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(tempMap.get(parts[i]), (field.getName().equals("types") || field.getName().equals("uri"))));
// }
// }
// }
// } else {
// if (stringObjectMap.containsKey(value)) {
// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri"))));
// }
// }
// }
// }
// } catch (IllegalAccessException | InvocationTargetException e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// });
// });
// return parsedData;
// }
//
// private static String normalizeValue(Object value, boolean jsonString) {
// if (value instanceof JSONArray) {
// if (jsonString) {
// return ((JSONArray)value).toJSONString();
// }
// JSONArray jarr = (JSONArray) value;
// if (jarr.get(0) instanceof String) {
// return jarr.get(0).toString();
// } else {
// for (Object o : jarr) {
// if ((o instanceof Map) && ((Map) o).containsKey("content")) {
// try {
// return ((Map<String, String>) o).get("content");
// }
// catch (ClassCastException e){
// if(((Map<?, ?>) o).get("content") instanceof Integer) {
// return String.valueOf(((Map<?, ?>) o).get("content"));
// }
// return null;
// }
// }
// }
// }
// } else if (value instanceof Map) {
// String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
// return ((Map<String, String>)value).get(key);
// }
// return value != null ? value.toString() : null;
// }
//}

57
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/AuthenticationConfiguration.java
View File

@ -1,57 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
public class AuthenticationConfiguration {
private String authUrl;
private String authMethod = "GET";
private String authTokenPath;
private String authRequestBody;
private String type;
public String getAuthUrl() {
return authUrl;
}
@XmlElement(name = "authUrl")
public void setAuthUrl(String authUrl) {
this.authUrl = authUrl;
}
public String getAuthMethod() {
return authMethod;
}
@XmlElement(name = "authUrlMethod")
public void setAuthMethod(String authMethod) {
this.authMethod = authMethod;
}
public String getAuthTokenPath() {
return authTokenPath;
}
@XmlElement(name = "authTokenJpath")
public void setAuthTokenPath(String authTokenPath) {
this.authTokenPath = authTokenPath;
}
public String getAuthRequestBody() {
return authRequestBody;
}
@XmlElement(name = "authUrlBody")
public void setAuthRequestBody(String authRequestBody) {
this.authRequestBody = authRequestBody;
}
public String getType() {
return type;
}
@XmlElement(name = "authType")
public void setType(String type) {
this.type = type;
}
}

134
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataFieldsUrlConfiguration.java
View File

@ -1,134 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
/**
* Created by ikalyvas on 6/29/2018.
*/
public class DataFieldsUrlConfiguration {
private String id;
private String name;
private String pid;
private String pidTypeField;
private String uri;
private String description;
private String source;
private String count;
private String path;
private String host;
private String types;
private String firstName;
private String lastName;
public String getId() {
return id;
}
@XmlElement(name = "id")
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
@XmlElement(name = "name")
public void setName(String name) {
this.name = name;
}
public String getPid() {
return pid;
}
@XmlElement(name = "pid")
public void setPid(String pid) {
this.pid = pid;
}
public String getPidTypeField() {
return pidTypeField;
}
@XmlElement(name = "pidTypeField")
public void setPidTypeField(String pidTypeField) {
this.pidTypeField = pidTypeField;
}
public String getUri() {
return uri;
}
@XmlElement(name = "uri")
public void setUri(String uri) {
this.uri = uri;
}
public String getDescription() {
return description;
}
@XmlElement(name = "description")
public void setDescription(String description) {
this.description = description;
}
public String getSource() {
return source;
}
@XmlElement(name = "source")
public void setSource(String source) {
this.source = source;
}
public String getCount() {
return count;
}
@XmlElement(name = "count")
public void setCount(String count) {
this.count = count;
}
public String getPath() {
return path;
}
@XmlElement(name = "path")
public void setPath(String path) {
this.path = path;
}
public String getHost() {
return host;
}
@XmlElement(name = "host")
public void setHost(String host) {
this.host = host;
}
@XmlElement(name = "types")
public String getTypes() {
return types;
}
public void setTypes(String types) {
this.types = types;
}
@XmlElement(name = "firstName")
public String getFirstName() {
return firstName;
}
public void setFirstName(String firstName) {
this.firstName = firstName;
}
@XmlElement(name = "lastName")
public String getLastName() {
return lastName;
}
public void setLastName(String lastName) {
this.lastName = lastName;
}
}

82
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataUrlConfiguration.java
View File

@ -1,82 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
/**
* Created by ikalyvas on 6/29/2018.
*/
public class DataUrlConfiguration {
private String path;
private DataFieldsUrlConfiguration fieldsUrlConfiguration;
private UrlConfiguration urlConfiguration;
private String parseClass;
private String parseField;
private List<String> mergedFields;
private String mergedFieldName;
public String getPath() {
return path;
}
@XmlElement(name = "path")
public void setPath(String path) {
this.path = path;
}
public DataFieldsUrlConfiguration getFieldsUrlConfiguration() {
return fieldsUrlConfiguration;
}
@XmlElement(name = "fields")
public void setFieldsUrlConfiguration(DataFieldsUrlConfiguration fieldsUrlConfiguration) {
this.fieldsUrlConfiguration = fieldsUrlConfiguration;
}
public UrlConfiguration getUrlConfiguration() {
return urlConfiguration;
}
@XmlElement(name = "urlConfig")
public void setUrlConfiguration(UrlConfiguration urlConfiguration) {
this.urlConfiguration = urlConfiguration;
}
public String getParseClass() {
return parseClass;
}
@XmlElement(name = "parse-class")
public void setParseClass(String parseClass) {
this.parseClass = parseClass;
}
public String getParseField() {
return parseField;
}
@XmlElement(name = "parse-field")
public void setParseField(String parseField) {
this.parseField = parseField;
}
public List<String> getMergedFields() {
return mergedFields;
}
@XmlElementWrapper(name = "merge-fields")
@XmlElement(name = "field")
public void setMergedFields(List<String> mergedFields) {
this.mergedFields = mergedFields;
}
public String getMergedFieldName() {
return mergedFieldName;
}
@XmlElement(name = "merge-field-name")
public void setMergedFieldName(String mergedFieldName) {
this.mergedFieldName = mergedFieldName;
}
}

173
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java
View File

@ -17,153 +17,8 @@ public class ExternalUrls implements Serializable {
Long maxresults; Long maxresults;
GrantUrls grants;
ProjectUrls projects;
RegistryUrls registries;
RepositoryUrls repositories;
PubRepositoryUrls pubRepositories;
JournalUrls journals;
TaxonomiesUrls taxonomies;
PublicationsUrls publications;
ServiceUrls services;
ResearcherUrls researchers;
OrganisationUrls organisations;
DatasetUrls datasets;
/*TagUrls tags;*/
FunderUrls funders;
LicenseUrls licenses;
ValidationUrls validations;
Map<String, PrefillingConfig> prefillings; Map<String, PrefillingConfig> prefillings;
public RegistryUrls getRegistries() {
return registries;
}
@XmlElement(name = "registries")
public void setRegistries(RegistryUrls registries) {
this.registries = registries;
}
public RepositoryUrls getRepositories() {
return repositories;
}
@XmlElement(name = "repositories")
public void setRepositories(RepositoryUrls repositories) {
this.repositories = repositories;
}
public PubRepositoryUrls getPubRepositories() {
return pubRepositories;
}
public void setPubRepositories(PubRepositoryUrls pubRepositories) {
this.pubRepositories = pubRepositories;
}
public void setJournals(JournalUrls journals) {
this.journals = journals;
}
public JournalUrls getJournals() {
return journals;
}
public JournalUrls getJournal() {
return journals;
}
@XmlElement(name = "journals")
public void setJournal(JournalUrls journal) {
this.journals = journal;
}
public TaxonomiesUrls getTaxonomies() {
return taxonomies;
}
@XmlElement(name = "taxonomies")
public void setTaxonomies(TaxonomiesUrls taxonomies) {
this.taxonomies = taxonomies;
}
public PublicationsUrls getPublications() {
return publications;
}
public void setPublications(PublicationsUrls publications) {
this.publications = publications;
}
public ServiceUrls getServices() {
return services;
}
@XmlElement(name = "services")
public void setServices(ServiceUrls services) {
this.services = services;
}
public ResearcherUrls getResearchers() {
return researchers;
}
@XmlElement(name = "researchers")
public void setResearchers(ResearcherUrls researchers) {
this.researchers = researchers;
}
/*public TagUrls getTags() {
return tags;
}
@XmlElement(name = "tags")
public void setTags(TagUrls tags) {
this.tags = tags;
}*/
public OrganisationUrls getOrganisations() {
return organisations;
}
@XmlElement(name = "organisations")
public void setOrganisations(OrganisationUrls organisations) {
this.organisations = organisations;
}
public GrantUrls getGrants() {
return grants;
}
@XmlElement(name = "grants")
public void setGrants(GrantUrls grants) {
this.grants = grants;
}
public ProjectUrls getProjects() {
return projects;
}
@XmlElement(name = "projects")
public void setProjects(ProjectUrls projects) {
this.projects = projects;
}
public FunderUrls getFunders() {
return funders;
}
public void setFunders(FunderUrls funders) {
this.funders = funders;
}
public Long getMaxresults() { public Long getMaxresults() {
return maxresults; return maxresults;
} }
@ -173,34 +28,6 @@ public class ExternalUrls implements Serializable {
this.maxresults = maxresults; this.maxresults = maxresults;
} }
public DatasetUrls getDatasets() {
return datasets;
}
@XmlElement(name = "datasets")
public void setDatasets(DatasetUrls datasets) {
this.datasets = datasets;
}
public LicenseUrls getLicenses() {
return licenses;
}
@XmlElement(name = "licenses")
public void setLicenses(LicenseUrls licenses) {
this.licenses = licenses;
}
public ValidationUrls getValidations() {
return validations;
}
@XmlElement(name = "validators")
public void setValidations(ValidationUrls validations) {
this.validations = validations;
}
public Map<String, PrefillingConfig> getPrefillings() { public Map<String, PrefillingConfig> getPrefillings() {
return prefillings; return prefillings;
} }

48
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/QueryConfig.java
View File

@ -1,48 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
public class QueryConfig {
private String condition;
private String separator;
private String value;
private Integer ordinal;
public String getCondition() {
return condition;
}
@XmlElement(name = "condition")
public void setCondition(String condition) {
this.condition = condition;
}
public String getSeparator() {
return separator;
}
@XmlElement(name = "separator")
public void setSeparator(String separator) {
this.separator = separator;
}
public String getValue() {
return value;
}
@XmlElement(name = "value")
public void setValue(String value) {
this.value = value;
}
public Integer getOrdinal() {
return ordinal;
}
@XmlElement(name = "ordinal")
public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal;
}
}

128
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/UrlConfiguration.java
View File

@ -1,34 +1,23 @@
package eu.eudat.service.remotefetcher.config; package eu.eudat.service.remotefetcher.config;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public class UrlConfiguration { public class UrlConfiguration implements SourceBaseConfiguration {
private String key; private String key;
private String label; private String label;
private Integer ordinal; private Integer ordinal;
private String url; private ReferenceTypeSourceType type;
private boolean dataInListing;
private DataUrlConfiguration data;
private String type;
private String paginationPath;
private String contentType;
private String funderQuery;
private String firstpage;
private String requestType = "GET";
private String requestBody = "";
private String filterType = "remote";
private AuthenticationConfiguration auth;
private List<QueryConfig> queries;
public String getKey() { public String getKey() {
return key; return key;
} }
@XmlElement(name = "key")
public void setKey(String key) { public void setKey(String key) {
this.key = key; this.key = key;
} }
@ -36,121 +25,24 @@ public class UrlConfiguration {
public String getLabel() { public String getLabel() {
return label; return label;
} }
@XmlElement(name = "label")
public void setLabel(String label) { public void setLabel(String label) {
this.label = label; this.label = label;
} }
public String getUrl() {
return url;
}
@XmlElement(name = "url")
public void setUrl(String url) {
this.url = url;
}
public boolean isDataInListing() {
return dataInListing;
}
@XmlElement(name = "dataInListing")
public void setDataInListing(boolean dataInListing) {
this.dataInListing = dataInListing;
}
public Integer getOrdinal() { public Integer getOrdinal() {
return ordinal; return ordinal;
} }
@XmlElement(name = "ordinal")
public void setOrdinal(Integer ordinal) { public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal; this.ordinal = ordinal;
} }
public DataUrlConfiguration getData() { public ReferenceTypeSourceType getType() {
return data;
}
@XmlElement(name = "data")
public void setData(DataUrlConfiguration data) {
this.data = data;
}
public String getPaginationPath() {
return paginationPath;
}
@XmlElement(name = "paginationpath")
public void setPaginationPath(String paginationPath) {
this.paginationPath = paginationPath;
}
public String getType() {
return type; return type;
} }
@XmlElement(name = "type")
public void setType(String type) { public void setType(ReferenceTypeSourceType type) {
this.type = type; this.type = type;
} }
public String getContentType() {
return contentType;
}
@XmlElement(name = "contenttype")
public void setContentType(String contentType) {
this.contentType = contentType;
}
public String getFunderQuery() {
return funderQuery;
}
@XmlElement(name = "funderQuery")
public void setFunderQuery(String funderQuery) {
this.funderQuery = funderQuery;
}
public String getFirstpage() {
return firstpage;
}
@XmlElement(name = "firstPage")
public void setFirstpage(String firstpage) {
this.firstpage = firstpage;
}
public String getRequestType() {
return requestType;
}
@XmlElement(name = "request")
public void setRequestType(String requestType) {
this.requestType = requestType != null ? requestType : "GET";
}
public String getRequestBody() {
return requestBody;
}
@XmlElement(name = "requestBody")
public void setRequestBody(String requestBody) {
this.requestBody = requestBody != null ? requestBody : "";
}
public String getFilterType() {
return filterType;
}
@XmlElement(name = "filterType")
public void setFilterType(String filterType) {
this.filterType = filterType;
}
public List<QueryConfig> getQueries() {
return queries;
}
@XmlElementWrapper
@XmlElement(name = "query")
public void setQueries(List<QueryConfig> queries) {
this.queries = queries;
}
public AuthenticationConfiguration getAuth() {
return auth;
}
@XmlElement(name="authentication")
public void setAuth(AuthenticationConfiguration auth) {
this.auth = auth;
}
} }

18
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/AuthenticationConfiguration.java Normal file
View File

@ -0,0 +1,18 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
public interface AuthenticationConfiguration {
Boolean getEnabled();
String getAuthUrl();
ReferenceTypeExternalApiHTTPMethodType getAuthMethod();
String getAuthTokenPath();
String getAuthRequestBody();
String getType();
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/DatasetUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class DatasetUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/FunderUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class FunderUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

31
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GeneralUrls.java
View File

@ -1,31 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import java.util.ArrayList;
import java.util.List;
public class GeneralUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public GeneralUrls() {
this.urls = new ArrayList<>();
}
@Override
public List<UrlConfiguration> getUrls() {
return urls;
}
@Override
public FetchStrategy getFetchMode() {
return fetchMode;
}
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

5
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java
View File

@ -1,12 +1,13 @@
package eu.eudat.service.remotefetcher.config.entities; package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import java.util.List; import java.util.List;
public abstract class GenericUrls { public abstract class GenericUrls {
public abstract List<UrlConfiguration> getUrls(); public abstract List<SourceBaseConfiguration> getUrls();
public abstract FetchStrategy getFetchMode(); public abstract FetchStrategy getFetchMode();
} }

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GrantUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class GrantUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/JournalUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class JournalUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/LicenseUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class LicenseUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/OrganisationUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class OrganisationUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

4
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java
View File

@ -20,8 +20,8 @@ public class PrefillingSearch extends GenericUrls {
} }
@Override @Override
public List<UrlConfiguration> getUrls() { public List<SourceBaseConfiguration> getUrls() {
List<UrlConfiguration> urls = new ArrayList<>(); List<SourceBaseConfiguration> urls = new ArrayList<>();
urls.add(urlConfig); urls.add(urlConfig);
return urls; return urls;
} }

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ProjectUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ProjectUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PubRepositoryUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class PubRepositoryUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PublicationsUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class PublicationsUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

11
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/QueryConfig.java Normal file
View File

@ -0,0 +1,11 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface QueryConfig {
String getCondition();
String getSeparator();
String getValue();
Integer getOrdinal();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RegistryUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class RegistryUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RepositoryUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class RepositoryUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResearcherUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ResearcherUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

6
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultFieldsMappingConfiguration.java Normal file
View File

@ -0,0 +1,6 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface ResultFieldsMappingConfiguration {
String getResponsePath();
String getCode();
}

9
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultsConfiguration.java Normal file
View File

@ -0,0 +1,9 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
public interface ResultsConfiguration <T extends ResultFieldsMappingConfiguration> {
String getResultsArrayPath();
List<T> getFieldsMapping();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ServiceUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ServiceUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

10
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceBaseConfiguration.java Normal file
View File

@ -0,0 +1,10 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
public interface SourceBaseConfiguration {
String getKey();
String getLabel();
Integer getOrdinal();
ReferenceTypeSourceType getType();
}

29
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceExternalApiConfiguration.java Normal file
View File

@ -0,0 +1,29 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import java.util.List;
public interface SourceExternalApiConfiguration<RsConfig extends ResultsConfiguration<? extends ResultFieldsMappingConfiguration>, AuthConfig extends AuthenticationConfiguration, QConfig extends QueryConfig> extends SourceBaseConfiguration {
String getUrl();
RsConfig getResults();
String getPaginationPath();
String getContentType();
String getFirstPage();
ReferenceTypeExternalApiHTTPMethodType getHttpMethod();
String getRequestBody();
String getFilterType();
AuthConfig getAuth();
List<QConfig> getQueries();
}

7
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceStaticOptionConfiguration.java Normal file
View File

@ -0,0 +1,7 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
public interface SourceStaticOptionConfiguration <Option extends StaticOption> extends SourceBaseConfiguration {
List<Option> getOptions();
}

6
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/StaticOption.java Normal file
View File

@ -0,0 +1,6 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface StaticOption {
String getCode();
String getValue();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/TagUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
/**
* Created by ikalyvas on 7/9/2018.
*/
public class TagUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/TaxonomiesUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class TaxonomiesUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ValidationUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
public class ValidationUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

46
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/models/ExternalDataResult.java Normal file
View File

@ -0,0 +1,46 @@
package eu.eudat.service.remotefetcher.models;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class ExternalDataResult {
List<Map<String, String>> results;
List<Map<String, Object>> rawData;
public ExternalDataResult() {
this.results = new ArrayList<>();
}
public ExternalDataResult(List<Map<String, String>> results) {
this.results = results;
}
public List<Map<String, String>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
this.results = results;
}
public List<Map<String, Object>> getRawData() {
return rawData;
}
public void setRawData(List<Map<String, Object>> rawData) {
this.rawData = rawData;
}
public void addAll(ExternalDataResult other){
if (other == null) return;
if (other.getResults() != null){
if (this.getResults() == null) this.setResults(new ArrayList<>());
this.getResults().addAll(other.getResults());
}
if (other.getRawData() != null){
if (this.getRawData() == null) this.setRawData(new ArrayList<>());
this.getRawData().addAll(other.getRawData());
}
}
}

37
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/models/ExternalRefernceResult.java
View File

@ -1,37 +0,0 @@
package eu.eudat.service.remotefetcher.models;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ExternalRefernceResult {
List<Map<String, String>> results;
Map<String, Integer> pagination;
public ExternalRefernceResult() {
this.results = new ArrayList<>();
this.pagination = new HashMap<>();
}
public ExternalRefernceResult(List<Map<String, String>> results, Map<String, Integer> pagination) {
this.results = results;
this.pagination = pagination;
}
public List<Map<String, String>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
this.results = results;
}
public Map<String, Integer> getPagination() {
return pagination;
}
public void setPagination(Map<String, Integer> pagination) {
this.pagination = pagination;
}
}

5
dmp-backend/web/src/main/java/eu/eudat/controllers/v2/ReferenceController.java
View File

@ -11,7 +11,6 @@ import eu.eudat.model.censorship.ReferenceCensor;
import eu.eudat.model.persist.ReferencePersist; import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.model.result.QueryResult; import eu.eudat.model.result.QueryResult;
import eu.eudat.query.ReferenceQuery; import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceLookup; import eu.eudat.query.lookup.ReferenceLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup; import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.reference.ReferenceService; import eu.eudat.service.reference.ReferenceService;
@ -95,12 +94,12 @@ public class ReferenceController {
@PostMapping("search") @PostMapping("search")
public List<Reference> searchReferenceWithDefinition(@RequestBody ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException { public List<Reference> searchReferenceWithDefinition(@RequestBody ReferenceSearchLookup lookup) throws MyNotFoundException {
logger.debug("search with db definition {}", Reference.class.getSimpleName()); logger.debug("search with db definition {}", Reference.class.getSimpleName());
this.censorFactory.censor(ReferenceCensor.class).censor(lookup.getProject(), null); this.censorFactory.censor(ReferenceCensor.class).censor(lookup.getProject(), null);
List<Reference> references = this.referenceService.searchReferenceWithDefinition(lookup); List<Reference> references = this.referenceService.searchReferenceData(lookup);
this.auditService.track(AuditableAction.Reference_Search, "lookup", lookup); this.auditService.track(AuditableAction.Reference_Search, "lookup", lookup);