MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

reference fetcher changes

This commit is contained in:
Efstratios Giannopoulos 2024-02-12 17:43:41 +02:00
parent 87dd56ff2c
commit 957c1fad58
54 changed files with 923 additions and 2591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/AuthenticationConfigurationEntity.java
View File

@ -1,9 +1,10 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.service.remotefetcher.config.entities.AuthenticationConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
public class AuthenticationConfigurationEntity { public class AuthenticationConfigurationEntity implements AuthenticationConfiguration {
private Boolean enabled; private Boolean enabled;
private String authUrl; private String authUrl;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/QueryConfigEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.QueryConfig;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
public class QueryConfigEntity { public class QueryConfigEntity implements QueryConfig {
private String condition; private String condition;
private String separator; private String separator;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceBaseConfigurationEntity.java
View File

@ -1,12 +1,13 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeSourceType; import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public abstract class ReferenceTypeSourceBaseConfigurationEntity { public abstract class ReferenceTypeSourceBaseConfigurationEntity implements SourceBaseConfiguration {
private String key; private String key;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceExternalApiConfigurationEntity.java
View File

@ -2,11 +2,12 @@ package eu.eudat.commons.types.referencetype;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.service.remotefetcher.config.entities.SourceExternalApiConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity { public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceExternalApiConfiguration<ResultsConfigurationEntity, AuthenticationConfigurationEntity, QueryConfigEntity> {
private String url; private String url;
private ResultsConfigurationEntity results; private ResultsConfigurationEntity results;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceStaticOptionConfigurationEntity.java
View File

@ -1,11 +1,12 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.SourceStaticOptionConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity { public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceStaticOptionConfiguration<ReferenceTypeStaticOptionEntity> {
List<ReferenceTypeStaticOptionEntity> options; List<ReferenceTypeStaticOptionEntity> options;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeStaticOptionEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.StaticOption;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
public class ReferenceTypeStaticOptionEntity { public class ReferenceTypeStaticOptionEntity implements StaticOption {
private String code; private String code;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultFieldsMappingConfigurationEntity.java
View File

@ -1,8 +1,9 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.ResultFieldsMappingConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
public class ResultFieldsMappingConfigurationEntity { public class ResultFieldsMappingConfigurationEntity implements ResultFieldsMappingConfiguration {
private String code; private String code;
private String responsePath; private String responsePath;

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultsConfigurationEntity.java
View File

@ -1,11 +1,12 @@
package eu.eudat.commons.types.referencetype; package eu.eudat.commons.types.referencetype;
import eu.eudat.service.remotefetcher.config.entities.ResultsConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public class ResultsConfigurationEntity { public class ResultsConfigurationEntity implements ResultsConfiguration<ResultFieldsMappingConfigurationEntity> {
private String resultsArrayPath; private String resultsArrayPath;
private List<ResultFieldsMappingConfigurationEntity> fieldsMapping; private List<ResultFieldsMappingConfigurationEntity> fieldsMapping;

58
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/DefinitionSearchBuilder.java
View File

@ -1,58 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Definition;
//import gr.cite.tools.data.builder.BuilderFactory;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class DefinitionSearchBuilder extends BaseBuilder<Definition, Map<String, String>> {
//
// private final BuilderFactory builderFactory;
// private final ReferenceTypeProperties properties;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public DefinitionSearchBuilder(
// ConventionService conventionService, BuilderFactory builderFactory, ReferenceTypeProperties properties) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(DefinitionSearchBuilder.class)));
// this.builderFactory = builderFactory;
// this.properties = properties;
// }
//
// public DefinitionSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Definition> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Definition> models = new ArrayList<>();
// //for (Map<String, String> d : data) {
// Definition m = new Definition();
// m.setFields(this.builderFactory.builder(FieldSearchBuilder.class).authorize(this.authorize).build(null, data));
// models.add(m);
// //}
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//}

88
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/FieldSearchBuilder.java
View File

@ -1,88 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.commons.enums.ReferenceFieldDataType;
//import eu.eudat.configurations.referencetype.ReferenceTypeField;
//import eu.eudat.configurations.referencetype.ReferenceTypeProperties;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Field;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class FieldSearchBuilder extends BaseBuilder<Field, Map<String, String>> {
//
// private final ReferenceTypeProperties properties;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public FieldSearchBuilder(
// ConventionService conventionService, ReferenceTypeProperties properties) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(FieldSearchBuilder.class)));
// this.properties = properties;
// }
//
// public FieldSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Field> build(FieldSet fields, List< Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Field> models = new ArrayList<>();
// for (Map<String, String> d : data) {
//
// ReferenceType referenceType = ReferenceType.valueOf(d.getOrDefault("referenceType", null));
// List<ReferenceTypeField> typeFields = this.getPropertiesFields(referenceType);
// if (typeFields.isEmpty()){
// return new ArrayList<>();
// }
//
// for (ReferenceTypeField typeField: typeFields){
// Field m = new Field();
// m.setCode(typeField.getCode());
// m.setDataType(ReferenceFieldDataType.valueOf(typeField.getDataType()));
// m.setValue(d.getOrDefault(typeField.getCode(), null));
// models.add(m);
// }
// }
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//
// private List<ReferenceTypeField> getPropertiesFields(ReferenceType referenceType){
// switch (referenceType) {
// case Taxonomies: return properties.getTaxonomy().get("fields");
// case Licenses: return properties.getLicence().get("fields");
// case Publications: return properties.getPublication().get("fields");
// case Journals: return properties.getJournal().get("fields");
// case PubRepositories: return properties.getPubRepository().get("fields");
// case DataRepositories: return properties.getDataRepository().get("fields");
// case Registries: return properties.getRegistry().get("fields");
// case Services: return properties.getService().get("fields");
// case Organizations: return properties.getOrganisation().get("fields");
// case Datasets: return properties.getDataset().get("fields");
// case Funder: return properties.getFunder().get("fields");
// case Grants: return properties.getGrant().get("fields");
// case Project: return properties.getProject().get("fields");
// case Researcher: return properties.getResearcher().get("fields");
// default: return null;
// }
// }
//}

96
dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/ReferenceSearchBuilder.java
View File

@ -1,96 +0,0 @@
//package eu.eudat.model.builder.referencesearch;
//
//import eu.eudat.authorization.AuthorizationFlags;
//import eu.eudat.commons.enums.ReferenceSourceType;
//import eu.eudat.convention.ConventionService;
//import eu.eudat.model.Reference;
//import eu.eudat.model.builder.BaseBuilder;
//import eu.eudat.model.referencedefinition.Definition;
//import eu.eudat.model.referencedefinition.Field;
//import gr.cite.tools.data.builder.BuilderFactory;
//import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.fieldset.FieldSet;
//import gr.cite.tools.logging.DataLogEntry;
//import gr.cite.tools.logging.LoggerService;
//import org.slf4j.LoggerFactory;
//import org.springframework.beans.factory.annotation.Autowired;
//import org.springframework.beans.factory.config.ConfigurableBeanFactory;
//import org.springframework.context.annotation.Scope;
//import org.springframework.stereotype.Component;
//
//import java.util.*;
//
//@Component
//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE)
//public class ReferenceSearchBuilder extends BaseBuilder<Reference, Map<String, String>> {
//
// private final BuilderFactory builderFactory;
// private EnumSet<AuthorizationFlags> authorize = EnumSet.of(AuthorizationFlags.None);
//
// @Autowired
// public ReferenceSearchBuilder(
// ConventionService conventionService,
// BuilderFactory builderFactory) {
// super(conventionService, new LoggerService(LoggerFactory.getLogger(ReferenceSearchBuilder.class)));
// this.builderFactory = builderFactory;
// }
//
// public ReferenceSearchBuilder authorize(EnumSet<AuthorizationFlags> values) {
// this.authorize = values;
// return this;
// }
//
// @Override
// public List<Reference> build(FieldSet fields, List<Map<String, String>> data) throws MyApplicationException {
// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0));
// this.logger.trace(new DataLogEntry("requested fields", fields));
// if (data == null)
// return new ArrayList<>();
//
// List<Reference> models = new ArrayList<>();
// for (Map<String, String> d : data) {
// Reference m = new Reference();
// if (d.containsKey("id")) m.setId(UUID.fromString(d.getOrDefault("id", null)));
// m.setLabel(d.getOrDefault("name", null));
// m.setSourceType(ReferenceSourceType.External);
// String type = d.getOrDefault("referenceType", null);
// if (type != null) m.setType(ReferenceType.valueOf(type));
//
// Definition definition = this.builderFactory.builder(DefinitionSearchBuilder.class).authorize(this.authorize).build(null, d);
// m.setDefinition(definition);
//
// switch (m.getType()){
// case Researcher:
// case Organizations:
// case Funder:
// case Grants:
// case DataRepositories:
// case PubRepositories:
// case Journals:
// case Datasets:
// case Registries:
// case Services:{
// for (Field field : definition.getFields()) {
// if (field.getCode().equals("pid") && field.getValue() != null) {
// if (d.containsKey("key")) {
// m.setReference(d.get("key") + ":" + field.getValue());
// }
// }
// if (field.getCode().equals("tag") && field.getValue() != null){
// m.setSource(field.getValue());
// }
// }
// }
// }
//
// m.setDescription(d.getOrDefault("description", null));
// m.setAbbreviation(d.getOrDefault("abbreviation", null));
//
// models.add(m);
//
// }
// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0));
// return models;
// }
//
//}

8
dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java
View File

@ -20,6 +20,7 @@ import eu.eudat.model.descriptionproperties.PropertyDefinitionFieldSetItem;
import eu.eudat.model.persist.DescriptionProfilingRequest; import eu.eudat.model.persist.DescriptionProfilingRequest;
import eu.eudat.model.persist.DescriptionProfilingWithDataRequest; import eu.eudat.model.persist.DescriptionProfilingWithDataRequest;
import eu.eudat.service.remotefetcher.ExternalUrlConfigProvider; import eu.eudat.service.remotefetcher.ExternalUrlConfigProvider;
import eu.eudat.service.remotefetcher.RemoteFetcherService;
import eu.eudat.service.remotefetcher.config.entities.*; import eu.eudat.service.remotefetcher.config.entities.*;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.builder.BuilderFactory;
@ -66,6 +67,7 @@ public class PrefillingServiceImpl implements PrefillingService {
private final ExternalUrlConfigProvider externalUrlConfigProvider; private final ExternalUrlConfigProvider externalUrlConfigProvider;
private final XmlHandlingService xmlHandlingService; private final XmlHandlingService xmlHandlingService;
private final ValidatorFactory validatorFactory; private final ValidatorFactory validatorFactory;
private final RemoteFetcherService remoteFetcherService;
@Autowired @Autowired
public PrefillingServiceImpl( public PrefillingServiceImpl(
EntityManager entityManager, EntityManager entityManager,
@ -76,7 +78,8 @@ public class PrefillingServiceImpl implements PrefillingService {
JsonHandlingService jsonHandlingService, JsonHandlingService jsonHandlingService,
ExternalUrlConfigProvider externalUrlConfigProvider, ExternalUrlConfigProvider externalUrlConfigProvider,
XmlHandlingService xmlHandlingService, XmlHandlingService xmlHandlingService,
ValidatorFactory validatorFactory) { ValidatorFactory validatorFactory,
RemoteFetcherService remoteFetcherService) {
this.entityManager = entityManager; this.entityManager = entityManager;
this.builderFactory = builderFactory; this.builderFactory = builderFactory;
this.queryFactory = queryFactory; this.queryFactory = queryFactory;
@ -86,6 +89,7 @@ public class PrefillingServiceImpl implements PrefillingService {
this.externalUrlConfigProvider = externalUrlConfigProvider; this.externalUrlConfigProvider = externalUrlConfigProvider;
this.xmlHandlingService = xmlHandlingService; this.xmlHandlingService = xmlHandlingService;
this.validatorFactory = validatorFactory; this.validatorFactory = validatorFactory;
this.remoteFetcherService = remoteFetcherService;
} }
@Override @Override
@ -98,7 +102,7 @@ public class PrefillingServiceImpl implements PrefillingService {
List<Map<String, String>> map; List<Map<String, String>> map;
Map<String, PrefillingConfig> prefillingConfigs = this.externalUrlConfigProvider.getExternalUrls().getPrefillings(); Map<String, PrefillingConfig> prefillingConfigs = this.externalUrlConfigProvider.getExternalUrls().getPrefillings();
// for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {//TODO new reference logic // for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {//TODO new reference logic
// map = remoteFetcherService.getExternalGeneric(externalReferenceCriteria, prefillingConfig.getPrefillingSearch()); // map = remoteFetcherService.getExternalData(prefillingConfig.getPrefillingSearch().getUrls(), externalReferenceCriteria, prefillingConfig.getPrefillingSearch().getFetchMode());
// prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList()); // prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList());
// if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) { // if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) {
// List<Map<String, Object>> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch()); // List<Map<String, Object>> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch());

6
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java
View File

@ -2,10 +2,12 @@ package eu.eudat.service.reference;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.Reference; import eu.eudat.model.Reference;
import eu.eudat.model.persist.ReferencePersist; import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup; import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException; import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException; import gr.cite.tools.exception.MyNotFoundException;
@ -24,5 +26,5 @@ public interface ReferenceService {
void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException; void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException;
List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException; List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException;
} }

713
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java
View File

@ -1,18 +1,12 @@
package eu.eudat.service.reference; package eu.eudat.service.reference;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.DocumentContext;
import com.jayway.jsonpath.JsonPath;
import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission; import eu.eudat.authorization.Permission;
import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive; import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType; import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.commons.types.reference.DefinitionEntity; import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity; import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.commons.types.referencetype.*; import eu.eudat.commons.types.referencetype.*;
@ -20,17 +14,18 @@ import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceEntity; import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.Reference; import eu.eudat.model.Reference;
import eu.eudat.model.ReferenceType;
import eu.eudat.model.builder.ReferenceBuilder; import eu.eudat.model.builder.ReferenceBuilder;
import eu.eudat.model.deleter.ReferenceDeleter; import eu.eudat.model.deleter.ReferenceDeleter;
import eu.eudat.model.persist.ReferencePersist; import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.model.persist.referencedefinition.DefinitionPersist; import eu.eudat.model.persist.referencedefinition.DefinitionPersist;
import eu.eudat.model.persist.referencedefinition.FieldPersist; import eu.eudat.model.persist.referencedefinition.FieldPersist;
import eu.eudat.query.ReferenceQuery; import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.ReferenceTypeQuery;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup; import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import eu.eudat.service.remotefetcher.RemoteFetcherService;
import eu.eudat.service.remotefetcher.models.ExternalRefernceResult; import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory; import gr.cite.tools.data.deleter.DeleterFactory;
@ -45,28 +40,17 @@ import gr.cite.tools.logging.LoggerService;
import gr.cite.tools.logging.MapLogEntry; import gr.cite.tools.logging.MapLogEntry;
import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.JAXBException;
import net.minidev.json.JSONArray;
import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.context.MessageSource; import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.http.client.reactive.ReactorClientHttpConnector;
import org.springframework.http.codec.json.Jackson2JsonDecoder;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.reactive.function.client.WebClient;
import reactor.netty.http.client.HttpClient;
import javax.management.InvalidApplicationException; import javax.management.InvalidApplicationException;
import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerException;
import java.time.Instant; import java.time.Instant;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
@Service @Service
public class ReferenceServiceImpl implements ReferenceService { public class ReferenceServiceImpl implements ReferenceService {
@ -81,8 +65,7 @@ public class ReferenceServiceImpl implements ReferenceService {
private final QueryFactory queryFactory; private final QueryFactory queryFactory;
private final XmlHandlingService xmlHandlingService; private final XmlHandlingService xmlHandlingService;
private final WebClient client; public final RemoteFetcherService remoteFetcherService;
public ReferenceServiceImpl( public ReferenceServiceImpl(
EntityManager entityManager, EntityManager entityManager,
AuthorizationService authorizationService, AuthorizationService authorizationService,
@ -91,7 +74,7 @@ public class ReferenceServiceImpl implements ReferenceService {
ConventionService conventionService, ConventionService conventionService,
MessageSource messageSource, MessageSource messageSource,
QueryFactory queryFactory, QueryFactory queryFactory,
XmlHandlingService xmlHandlingService) { XmlHandlingService xmlHandlingService, RemoteFetcherService remoteFetcherService) {
this.entityManager = entityManager; this.entityManager = entityManager;
this.authorizationService = authorizationService; this.authorizationService = authorizationService;
this.deleterFactory = deleterFactory; this.deleterFactory = deleterFactory;
@ -100,11 +83,7 @@ public class ReferenceServiceImpl implements ReferenceService {
this.messageSource = messageSource; this.messageSource = messageSource;
this.queryFactory = queryFactory; this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService; this.xmlHandlingService = xmlHandlingService;
this.client = WebClient.builder().codecs(clientCodecConfigurer -> { this.remoteFetcherService = remoteFetcherService;
clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
}
).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
} }
@Override @Override
@ -178,380 +157,340 @@ public class ReferenceServiceImpl implements ReferenceService {
this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id)); this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id));
} }
// public List<FetcherReference> searchReference(ReferenceType externalType, String query, String type) throws HugeResultSet, MyNotFoundException, InvalidApplicationException { @Override
// ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); public List<Reference> searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException {
// int initialOffset = 0;
// List<Map<String, String>> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(externalType, externalUrlCriteria, type); if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
// initialOffset = lookup.getPage().getOffset();
// List<FetcherReference> list = this.fetchFromDb(externalType, query, type, remoteRepos); lookup.getPage().setOffset(0);
// }
// list.addAll(remoteRepos.stream().map(FetcherReference::fromRemoteModel).toList());
// list = list.stream().filter(x -> x.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList());
// list.sort(Comparator.comparing(FetcherReference::getName));
// return list;
// }
// @Override ReferenceTypeEntity data = this.entityManager.find(ReferenceTypeEntity.class, lookup.getTypeId());
// public List<Reference> searchReference(ReferenceSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException { if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{lookup.getTypeId(), ReferenceType.class.getSimpleName()}, LocaleContextHolder.getLocale()));
// int initialOffset = 0;
// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
// initialOffset = lookup.getPage().getOffset();
// lookup.getPage().setOffset(0);
// }
//
// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
//
// List<Map<String, String>> remoteRepos = remoteFetcherService.getReferences(lookup.getType(), externalReferenceCriteria, lookup.getKey());
//
// List<Reference> externalModels = this.builderFactory.builder(ReferenceSearchBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), remoteRepos);
// List<Reference> models = this.fetchFromDb(lookup);
//
// models.addAll(externalModels);
//
// if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).collect(Collectors.toList()); }
// models.sort(Comparator.comparing(Reference::getLabel));
//
// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
// models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList();
// }
//
// return models;
// }
private List<Reference> fetchFromDb(ReferenceSearchLookup lookup){ ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike());
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal);
ExternalDataResult remoteRepos = this.getReferenceData(data, externalReferenceCriteria, lookup.getKey());
List<Reference> externalModels = new ArrayList<>();
if (remoteRepos != null && this.conventionService.isListNullOrEmpty(remoteRepos.getResults())){
List<ReferenceEntity> referenceEntities = new ArrayList<>();
for (Map<String, String> result : remoteRepos.getResults()){
if (result == null || result.isEmpty()) continue;;
ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data, remoteRepos);
referenceEntities.add(referenceEntity);
}
externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities);
}
List<Reference> models = this.fetchReferenceFromDb(lookup);
for (Reference externalReference : externalModels){
if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null &&x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference);
}
if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).toList(); }
models.sort(Comparator.comparing(Reference::getLabel));
if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList();
}
return models;
}
@NotNull
private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) {
ReferenceEntity referenceEntity = new ReferenceEntity(); //TODO new reference logic
referenceEntity.setTypeId(data.getId());
referenceEntity.setIsActive(IsActive.Active);
referenceEntity.setReference(result.getOrDefault("key", null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("pid", null));
referenceEntity.setSource(result.getOrDefault("tag", null));
referenceEntity.setAbbreviation(result.getOrDefault("abbreviation", null));
referenceEntity.setDescription(result.getOrDefault("description", null));
referenceEntity.setLabel(result.getOrDefault("name", null));
referenceEntity.setSourceType(ReferenceSourceType.External);
DefinitionEntity definitionEntity = new DefinitionEntity();
definitionEntity.setFields(new ArrayList<>());
for (Map.Entry<String, String> resultValue : result.entrySet()){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode(resultValue.getKey());
fieldEntity.setValue(resultValue.getValue());
fieldEntity.setDataType(ReferenceFieldDataType.Text);
definitionEntity.getFields().add(fieldEntity);
}
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
return referenceEntity;
}
private List<Reference> fetchReferenceFromDb(ReferenceSearchLookup lookup){
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId());
List<ReferenceEntity> data = query.collectAs(lookup.getProject()); List<ReferenceEntity> data = query.collectAs(lookup.getProject());
return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data); return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data);
} }
// private List<FetcherReference> fetchFromDb(ReferenceType externalType, String query, String type, List<Map<String, String>> remoteRepos) throws InvalidApplicationException { private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) {
// List<FetcherReference> list = new LinkedList<>(); ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition());
// switch (externalType) { if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult();
// case DataRepositories:
// case PubRepositories:
// case Journals: {
// DataRepositoryCriteria criteria = new DataRepositoryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<DataRepository> dataRepositoryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao().getWithCriteria(criteria)).toList();
// list = dataRepositoryList.stream().map(item -> new FetcherReference().fromDataRepository(item)).collect(Collectors.toList());
// }
// }
// case Registries: {
// RegistryCriteria criteria = new RegistryCriteria();
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
// if (type.equals("")) {
// List<Registry> registryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao().getWithCriteria(criteria)).toList();
// list = registryList.stream().map(item -> new FetcherReference().fromRegistry(item)).collect(Collectors.toList());
// }
// }
// case Services:
// {
// ServiceCriteria criteria = new ServiceCriteria();
//
// if (!query.isEmpty()) criteria.setLike(query);
// criteria.setCreationUserId(this.userScope.getUserId());
//
// if (type.equals("")) {
// List<Service> serviceList = (this.apiContext.getOperationsContext().getDatabaseRepository().getServiceDao().getWithCriteria(criteria)).toList();
// list = serviceList.stream().map(item -> new FetcherReference().fromService(item)).collect(Collectors.toList());
// }
// }
// case Datasets:{
//
// ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build();
//
// criteria.setCreationUserId(this.userScope.getUserId());
// QueryableList<ExternalDataset> items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria);
//
// list = items.select(item -> new FetcherReference().fromDataset(item));
// }
// case Taxonomies:
// case Publications:
// case Licenses:
// break;
// }
//
// return list;
// }
@Override ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).toList(), externalReferenceCriteria, key, null);
public List<Reference> searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException { for (Map<String, String> result: results.getResults()) {
result.put("referenceType", referenceType.getName());
ReferenceTypeQuery query = this.queryFactory.query(ReferenceTypeQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(lookup.getReferenceTypeId());
List<ReferenceTypeEntity> datas = query.collectAs(lookup.getProject());
if (datas.size() != 1 ){
return null;
} }
ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, datas.get(0).getDefinition());
List<Map<String, String>> remoteRepos = this.getAll(definition.getSources(), lookup);
return null;
}
private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
List<Map<String, String>> results = new LinkedList<>();
if (sources == null || sources.isEmpty()) {
return results; return results;
} }
sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal)); // private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
// List<Map<String, String>> results = new LinkedList<>();
//
// if (sources == null || sources.isEmpty()) {
// return results;
// }
//
// sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
//
// List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
// apiSources.forEach(source -> {
// try {
// String auth = null;
// if (source.getAuth()!= null) {
// //auth = this.getAuthentication(source.getAuth());
// }
// results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// });
// List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
// staticSources.forEach(source -> {
// Map<String, String> map = new HashMap<>();
// source.getOptions().forEach(option -> {
// map.put(option.getCode(), option.getValue());
// map.put("tag", source.getLabel());
// map.put("key", source.getKey());
// });
// results.add(map);
// });
//
// return results;
// }
List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList(); // private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
apiSources.forEach(source -> { // HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
try { // Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
String auth = null; // .contentType(MediaType.APPLICATION_JSON)
if (source.getAuth()!= null) { // .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
//auth = this.getAuthentication(source.getAuth()); // .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
} // })).block();
results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth)); //
} catch (Exception e) { //
logger.error(e.getLocalizedMessage(), e); //
} // return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
}); // }
List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
staticSources.forEach(source -> {
Map<String, String> map = new HashMap<>();
source.getOptions().forEach(option -> {
map.put(option.getCode(), option.getValue());
map.put("tag", source.getLabel());
map.put("key", source.getKey());
});
results.add(map);
});
return results; // private String parseBodyString(String bodyString) {
} // String finalBodyString = bodyString;
// if (bodyString.contains("{env:")) {
// int index = bodyString.indexOf("{env: ");
// while (index >= 0) {
// int endIndex = bodyString.indexOf("}", index + 6);
// String envName = bodyString.substring(index + 6, endIndex);
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// index = bodyString.indexOf("{env: ", index + 6);
// }
// }
// return finalBodyString;
// }
private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) { // private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name()); // Set<Integer> pages = new HashSet<>();
Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl()) //
.contentType(MediaType.APPLICATION_JSON) // String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
.bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) // String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
.exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() { //
})).block(); // ExternalDataResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
// if(externalRefernceResult != null) {
// if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
// .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
// .collect(Collectors.toList()));
// }
// if (fetchStrategy == FetchStrategy.FIRST)
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
//
// //Long maxResults = configLoader.getExternalUrls().getMaxresults();
// Long maxResults = Long.valueOf(1000);
// if (externalRefernceResult.getResults().size() > maxResults)
// throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getResults().size() + " results... Please submit a more detailed search query");
//
// Optional<ExternalDataResult> optionalResults = pages.parallelStream()
// .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
// .filter(Objects::nonNull)
// .reduce((result1, result2) -> {
// result1.getResults().addAll(result2.getResults());
// return result1;
// });
// ExternalDataResult remainingExternalDataResult = optionalResults.orElseGet(ExternalDataResult::new);
// remainingExternalDataResult.getResults().addAll(externalRefernceResult.getResults());
//
// return remainingExternalDataResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
// }
// else {
// return new LinkedList<>();
// }
// }
// private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
// String completedPath = urlPath;
//
// if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
// if (lookup.getLike() != null) {
// completedPath = completedPath.replace("{like}", lookup.getLike());
// } else {
// completedPath = completedPath.replace("{like}", "*");
// }
// }
//
// if (urlPath.contains("{like}")){
// if (lookup.getLike() != null) {
// completedPath = completedPath.replace("{like}", lookup.getLike());
// } else {
// completedPath = completedPath.replace("{like}", "");
// }
// }
//
// if (urlPath.contains("{page}")) {
// if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
// completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
// } else if (firstPage != null) {
// completedPath = completedPath.replace("{page}", firstPage);
// } else {
// completedPath = completedPath.replace("{page}", "1");
// }
// }
//
// if (urlPath.contains("{pageSize}")){
// if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
// completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
// } else {
// completedPath = completedPath.replace("{pageSize}", "100");
// }
// }
//
//
// return completedPath;
// }
// protected ExternalDataResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
//
// try {
// ResponseEntity<String> response;
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
//
// response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
// if (contentType != null && !contentType.isEmpty()) {
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
// httpHeaders.setContentType(MediaType.valueOf(contentType));
// }
// if (auth != null) {
// httpHeaders.set("Authorization", auth);
// }
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
// if (response.getStatusCode() == HttpStatus.OK) { // success
// ExternalDataResult externalDataResult = new ExternalDataResult();
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
// externalDataResult = this.getFromJson(jsonContext, resultsEntity);
// }
//
// return externalDataResult;
// }
// } catch (Exception exception) {
// logger.error(exception.getMessage(), exception);
// }
//
// return null;
// }
// public static ExternalDataResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath()); // return new ExternalDataResult(parseData(jsonContext, resultsEntity));
} // }
private String parseBodyString(String bodyString) {
String finalBodyString = bodyString;
if (bodyString.contains("{env:")) {
int index = bodyString.indexOf("{env: ");
while (index >= 0) {
int endIndex = bodyString.indexOf("}", index + 6);
String envName = bodyString.substring(index + 6, endIndex);
finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
index = bodyString.indexOf("{env: ", index + 6);
}
}
return finalBodyString;
}
private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
Set<Integer> pages = new HashSet<>();
String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
if(externalRefernceResult != null) {
if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
if (fetchStrategy == FetchStrategy.FIRST)
return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set
for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++)
pages.add(i);
//Long maxResults = configLoader.getExternalUrls().getMaxresults();
Long maxResults = Long.valueOf(1000);
if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults))
throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query");
Optional<ExternalRefernceResult> optionalResults = pages.parallelStream()
.map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
.filter(Objects::nonNull)
.reduce((result1, result2) -> {
result1.getResults().addAll(result2.getResults());
return result1;
});
ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults());
return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
}
else {
return new LinkedList<>();
}
}
private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
String completedPath = urlPath;
if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
if (lookup.getLike() != null) {
completedPath = completedPath.replace("{like}", lookup.getLike());
} else {
completedPath = completedPath.replace("{like}", "*");
}
}
if (urlPath.contains("{like}")){
if (lookup.getLike() != null) {
completedPath = completedPath.replace("{like}", lookup.getLike());
} else {
completedPath = completedPath.replace("{like}", "");
}
}
if (urlPath.contains("{page}")) {
if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
} else if (firstPage != null) {
completedPath = completedPath.replace("{page}", firstPage);
} else {
completedPath = completedPath.replace("{page}", "1");
}
}
if (urlPath.contains("{pageSize}")){
if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
} else {
completedPath = completedPath.replace("{pageSize}", "100");
}
}
return completedPath; // private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
} // List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
// List<Map<String, String>> parsedData = new ArrayList<>();
protected ExternalRefernceResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) { //
// for (Map<String, String> stringObjectMap: rawData){
try { // Map<String, String> map = new HashMap<>();
ResponseEntity<String> response; // for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
JsonNode jsonBody = new ObjectMapper().readTree(requestBody); // String pathValue = field.getResponsePath();
// if (!pathValue.contains(".")){
response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> { // if (stringObjectMap.containsKey(pathValue)) {
if (contentType != null && !contentType.isEmpty()) { // //map.put(field.getCode(), stringObjectMap.get(pathValue));
httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); // map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
httpHeaders.setContentType(MediaType.valueOf(contentType)); // }
} // }else {
if (auth != null) { // if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
httpHeaders.set("Authorization", auth); // String value = null;
} // Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
}).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); // if (fieldObj != null){
if (response.getStatusCode() == HttpStatus.OK) { // success // if (fieldObj instanceof Map){
ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(); // Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
if (response.getHeaders().get("Content-Type").get(0).contains("json")) { // if(o instanceof String){
DocumentContext jsonContext = JsonPath.parse(response.getBody()); // value = (String)o;
externalRefernceResult = this.getFromJson(jsonContext, resultsEntity); // }
} // else if(o instanceof Integer){
// value = String.valueOf(o);
// }
if (externalRefernceResult.getPagination().size() == 0) { // } else if (fieldObj instanceof List) {
externalRefernceResult.getPagination().put("pages", 1); // Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size()); // if(o instanceof String){
} // value = (String)o;
return externalRefernceResult; // }
} // else if(o instanceof Integer){
} catch (Exception exception) { // value = String.valueOf(o);
logger.error(exception.getMessage(), exception); // }
} // }
// }
return null; // if (value != null){
} // map.put(field.getCode(), value);
// }
// }
public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { // }
return new ExternalRefernceResult(parseData(jsonContext, resultsEntity), // }
new HashMap<>(1, 1)); // parsedData.add(map);
} // }
// return parsedData;
// }
private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { //
List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath()); //
List<Map<String, String>> parsedData = new ArrayList<>(); // private static String normalizeValue(Object value) {
// if (value instanceof JSONArray) {
for (Map<String, String> stringObjectMap: rawData){ // JSONArray jarr = (JSONArray) value;
Map<String, String> map = new HashMap<>(); // if (jarr.get(0) instanceof String) {
for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){ // return jarr.get(0).toString();
String pathValue = field.getResponsePath(); // } else {
if (!pathValue.contains(".")){ // for (Object o : jarr) {
if (stringObjectMap.containsKey(pathValue)) { // if ((o instanceof Map) && ((Map) o).containsKey("content")) {
//map.put(field.getCode(), stringObjectMap.get(pathValue)); // try {
map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue))); // return ((Map<String, String>) o).get("content");
} // }
}else { // catch (ClassCastException e){
if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){ // if(((Map<?, ?>) o).get("content") instanceof Integer) {
String value = null; // return String.valueOf(((Map<?, ?>) o).get("content"));
Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]); // }
if (fieldObj != null){ // return null;
if (fieldObj instanceof Map){ // }
Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]); // }
if(o instanceof String){ // }
value = (String)o; // }
} // } else if (value instanceof Map) {
else if(o instanceof Integer){ // String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
value = String.valueOf(o); // return ((Map<String, String>)value).get(key);
} // }
} else if (fieldObj instanceof List) { // return value != null ? value.toString() : null;
Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]); // }
if(o instanceof String){
value = (String)o;
}
else if(o instanceof Integer){
value = String.valueOf(o);
}
}
}
if (value != null){
map.put(field.getCode(), value);
}
}
}
}
parsedData.add(map);
}
return parsedData;
}
private static String normalizeValue(Object value) {
if (value instanceof JSONArray) {
JSONArray jarr = (JSONArray) value;
if (jarr.get(0) instanceof String) {
return jarr.get(0).toString();
} else {
for (Object o : jarr) {
if ((o instanceof Map) && ((Map) o).containsKey("content")) {
try {
return ((Map<String, String>) o).get("content");
}
catch (ClassCastException e){
if(((Map<?, ?>) o).get("content") instanceof Integer) {
return String.valueOf(((Map<?, ?>) o).get("content"));
}
return null;
}
}
}
}
} else if (value instanceof Map) {
String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
return ((Map<String, String>)value).get(key);
}
return value != null ? value.toString() : null;
}
} }

1
dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java
View File

@ -18,7 +18,6 @@ import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl;
import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory; import gr.cite.tools.data.deleter.DeleterFactory;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException; import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException; import gr.cite.tools.exception.MyNotFoundException;

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java
View File

@ -1,21 +1,14 @@
//package eu.eudat.service.remotefetcher; package eu.eudat.service.remotefetcher;
//
//import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.data.ReferenceTypeEntity;
//import eu.eudat.service.remotefetcher.config.entities.GenericUrls; import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import gr.cite.tools.exception.MyNotFoundException; import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
// import eu.eudat.service.remotefetcher.models.ExternalDataResult;
//import java.util.List;
//import java.util.Map; import java.util.List;
//
//public interface RemoteFetcherService { public interface RemoteFetcherService {
// List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException; ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy);
// Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
// GenericUrls getExternalUrls(ReferenceType referenceType); }
//
// Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException;
//
// List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
//
// List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls);
//}

913
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java
View File

@ -1,533 +1,388 @@
//package eu.eudat.service.remotefetcher; package eu.eudat.service.remotefetcher;
//
//import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.core.type.TypeReference;
//import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
//import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
//import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.DocumentContext;
//import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.JsonPath;
//import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.commons.XmlHandlingService;
//import eu.eudat.service.remotefetcher.config.AuthenticationConfiguration; import eu.eudat.commons.enums.ReferenceTypeSourceType;
//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration; import eu.eudat.commons.exceptions.HugeResultSetException;
//import eu.eudat.service.remotefetcher.config.QueryConfig; import eu.eudat.convention.ConventionService;
//import eu.eudat.service.remotefetcher.config.UrlConfiguration; import eu.eudat.service.remotefetcher.config.entities.*;
//import eu.eudat.service.remotefetcher.config.entities.GenericUrls; import eu.eudat.service.remotefetcher.models.ExternalDataResult;
//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
//import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import gr.cite.tools.exception.MyApplicationException;
//import gr.cite.tools.exception.MyNotFoundException; import net.minidev.json.JSONArray;
//import jakarta.xml.bind.JAXBContext; import org.slf4j.Logger;
//import jakarta.xml.bind.Unmarshaller; import org.slf4j.LoggerFactory;
//import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired;
//import org.slf4j.LoggerFactory; import org.springframework.core.ParameterizedTypeReference;
//import org.springframework.beans.factory.annotation.Autowired; import org.springframework.http.*;
//import org.springframework.core.ParameterizedTypeReference; import org.springframework.http.client.reactive.ReactorClientHttpConnector;
//import org.springframework.http.*; import org.springframework.http.codec.json.Jackson2JsonDecoder;
//import org.springframework.http.client.reactive.ReactorClientHttpConnector; import org.springframework.stereotype.Service;
//import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.web.reactive.function.client.WebClient;
//import org.springframework.stereotype.Service; import reactor.netty.http.client.HttpClient;
//import org.springframework.web.client.RestTemplate;
//import org.springframework.web.reactive.function.client.WebClient; import java.io.File;
//import reactor.netty.http.client.HttpClient; import java.nio.file.Paths;
// import java.util.*;
//import java.io.File; import java.util.stream.Collectors;
//import java.io.StringReader;
//import java.lang.reflect.Method; @Service
//import java.nio.file.Paths; public class RemoteFetcherServiceImpl implements RemoteFetcherService {
//import java.util.*; private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
//import java.util.stream.Collectors;
// private WebClient webClient;
//@Service private final ExternalUrlConfigProvider externalUrlConfigProvider;
//public class RemoteFetcherServiceImpl implements RemoteFetcherService { private final ConventionService conventionService;
// private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class); private final XmlHandlingService xmlHandlingService;
// @Autowired
// private WebClient webClient; public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) {
// private final ExternalUrlConfigProvider externalUrlConfigProvider; this.externalUrlConfigProvider = externalUrlConfigProvider;
// @Autowired this.conventionService = conventionService;
// public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider) { this.xmlHandlingService = xmlHandlingService;
// this.externalUrlConfigProvider = externalUrlConfigProvider; }
// }
// private WebClient getWebClient(){
// private WebClient getWebClient(){ if (this.webClient == null) {
// if (this.webClient == null) { this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> {
// this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> { clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON));
// clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON)); clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here???
// clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here??? }
// } ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build();
// ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build(); }
// } return webClient;
// return webClient; }
// }
//
// @Override
// public List<Map<String, String>> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException { @Override
// FetchStrategy fetchStrategy = null; public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) {
// List<SourceBaseConfiguration> apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList();
//// GenericUrls exGenericUrls = this.getExternalUrls(referenceType); if (!this.conventionService.isNullOrEmpty(key)){
// List<UrlConfiguration> urlConfigs = key != null && !key.isEmpty() ? exGenericUrls.getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList();
// : exGenericUrls.getUrls(); }
// if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult();
// List<Map<String, String>> results = getAll(urlConfigs, fetchStrategy, externalReferenceCriteria);
// for (Map<String, String> result: results) { apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal));
// result.put("referenceType", referenceType.name());
// } return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria);
// return results; }
// }
// @Override
// @Override public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
// public GenericUrls getExternalUrls(ReferenceType referenceType) { return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size();
// return switch (referenceType) { }
// case Taxonomies -> this.externalUrlConfigProvider.getExternalUrls().getTaxonomies();
// case Licenses -> this.externalUrlConfigProvider.getExternalUrls().getLicenses(); private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
// case Publications -> this.externalUrlConfigProvider.getExternalUrls().getPublications();
// case Journals -> this.externalUrlConfigProvider.getExternalUrls().getJournals(); ExternalDataResult results = new ExternalDataResult();
// case PubRepositories -> this.externalUrlConfigProvider.getExternalUrls().getPubRepositories();
// case DataRepositories -> this.externalUrlConfigProvider.getExternalUrls().getRepositories(); if (this.conventionService.isListNullOrEmpty(sources)) return new ExternalDataResult();
// case Registries -> this.externalUrlConfigProvider.getExternalUrls().getRegistries();
// case Services -> this.externalUrlConfigProvider.getExternalUrls().getServices(); for (SourceBaseConfiguration source : sources) {
// case Grants -> this.externalUrlConfigProvider.getExternalUrls().getGrants(); if (source.getType() == null || source.getType().equals(ReferenceTypeSourceType.API)) {
// case Organizations -> this.externalUrlConfigProvider.getExternalUrls().getOrganisations(); try {
// case Datasets -> this.externalUrlConfigProvider.getExternalUrls().getDatasets(); SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource = (SourceExternalApiConfiguration)source;
// case Funder -> this.externalUrlConfigProvider.getExternalUrls().getFunders(); this.applyFunderToQuery(apiSource, externalReferenceCriteria);
// case Project -> this.externalUrlConfigProvider.getExternalUrls().getProjects();
// case Researcher -> this.externalUrlConfigProvider.getExternalUrls().getResearchers(); String auth = null;
// default -> throw new IllegalArgumentException("Type not found" + referenceType); if (apiSource.getAuth() != null) {
// }; auth = this.buildAuthentication(apiSource.getAuth());
// } }
// results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth));
// @Override } catch (Exception e) {
// public Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException { logger.error(e.getLocalizedMessage(), e);
// List<UrlConfiguration> urlConfigs = }
// key != null && !key.isEmpty() ? this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) } else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) {
// : this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls(); SourceStaticOptionConfiguration<StaticOption> staticSource = (SourceStaticOptionConfiguration)source;
// FetchStrategy fetchStrategy = this.externalUrlConfigProvider.getExternalUrls().getValidations().getFetchMode(); results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO:
// List<Map<String, String>> data = this.getAll(urlConfigs, fetchStrategy, externalReferenceCriteria); }
// return data.size(); }
// } return results;
// }
// @Override
// public List<Map<String, String>> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) { private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) {
// List<UrlConfiguration> urlConfigurations = genericUrls.getUrls(); HttpMethod method;
// FetchStrategy fetchStrategy = genericUrls.getFetchMode(); switch (authenticationConfiguration.getAuthMethod()) {
// return getAll(urlConfigurations, fetchStrategy, externalReferenceCriteria); case GET -> method = HttpMethod.GET;
// } case POST -> method =HttpMethod.POST;
// default -> throw new MyApplicationException("unrecognized type " + authenticationConfiguration.getAuthMethod());
// @Override }
// public List<Map<String, Object>> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) {
// List<UrlConfiguration> urlConfigurations = genericUrls.getUrls(); Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
// return getAllWithData(urlConfigurations, externalReferenceCriteria); .contentType(MediaType.APPLICATION_JSON)
// } .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
// private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) { })).block();
// if (response == null) throw new MyApplicationException("Authentication " + authenticationConfiguration.getAuthUrl() + " failed");
// List<Map<String, String>> results = new LinkedList<>();
// return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null);
// if (urlConfigs == null || urlConfigs.isEmpty()) { }
// return results;
// }
//// throw new MyNotFoundException("No Repository urls found in configuration"); private void applyFunderToQuery(SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) {
// //TODO new reference logic
// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal)); // if (apiSource.getFunderQuery() != null) {
// for (UrlConfiguration urlConfiguration : urlConfigs) { // if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) {
// applyFunderQuery(urlConfiguration, externalReferenceCriteria); // apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery()));
// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
// try {
// String auth = null;
// if (urlConfiguration.getAuth() != null) {
// auth = this.getAuthentication(urlConfiguration.getAuth());
// }
// results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalReferenceCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType(), urlConfiguration.getQueries(), auth));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// } else if (urlConfiguration.getType() != null && urlConfiguration.getType().equals("Internal")) {
// results.addAll(getAllResultsFromMockUpJson(urlConfiguration.getUrl(), externalReferenceCriteria.getLike()));
// }
// }
// /* for (UrlConfiguration urlConfig : urlConfigs) {
// ifFunderQueryExist(urlConfig, externalUrlCriteria);
// if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
// results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType()));
// } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
// results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
// }
// }*/
// return results;
// }
//
// private String getAuthentication(AuthenticationConfiguration authenticationConfiguration) {
// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod());
// Map<String, Object> response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl())
// .contentType(MediaType.APPLICATION_JSON)
// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
// })).block();
//
//
//
// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
// }
//
// private List<Map<String, Object>> getAllWithData(List<UrlConfiguration> urlConfigs, ExternalReferenceCriteria externalReferenceCriteria) {
//
// List<Map<String, Object>> results = new LinkedList<>();
//
// if (urlConfigs == null || urlConfigs.isEmpty()) {
// return results;
// }
//
// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
// urlConfigs.forEach(urlConfiguration -> {
// applyFunderQuery(urlConfiguration, externalReferenceCriteria);
// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
// try {
// results.addAll(getAllResultsFromUrlWithData(urlConfiguration.getUrl(), urlConfiguration.getData(), externalReferenceCriteria, urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getQueries()));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// }
// });
// return results;
//
// }
//
// private void applyFunderQuery(UrlConfiguration urlConfiguration, ExternalReferenceCriteria externalReferenceCriteria) {
// if (urlConfiguration.getFunderQuery() != null) {
// if (externalReferenceCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) {
// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery()));
// } // }
// else { // else {
// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", "")); // apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", ""));
// } // }
// } // }
// } }
//
// private String calculateQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) { private String replaceLookupFieldQuery(ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig> queryConfigs) {
// String finalQuery = ""; String finalQuery = "";
// QueryConfig queryConfig = queryConfigs.stream().filter(queryConfigl -> externalReferenceCriteria.getLike().matches(queryConfigl.getCondition())) if (this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike())) return finalQuery;
// .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null); QueryConfig queryConfig = queryConfigs.stream().filter(x -> !this.conventionService.isNullOrEmpty(x.getCondition()) && externalReferenceCriteria.getLike().matches(x.getCondition()))
// if (queryConfig != null) { .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null);
// if (queryConfig.getSeparator() != null) { if (queryConfig != null) {
// String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator()); if (queryConfig.getSeparator() != null) {
// finalQuery = queryConfig.getValue(); String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator());
// for (int i = 0; i < likes.length; i++) { finalQuery = queryConfig.getValue();
// finalQuery = finalQuery.replaceAll("\\{like" + (i+1) + "}", likes[i]); for (int i = 0; i < likes.length; i++) {
// } finalQuery = finalQuery.replaceAll("\\{like" + (i + 1) + "}", likes[i]);
// } else { }
// finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike()); } else {
// } finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike());
// }
// }
// return finalQuery; }
// } return finalQuery;
// }
// protected String replaceCriteriaOnUrl(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
// String completedPath = path; protected String replaceLookupFields(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List<QueryConfig> queries) {
// if (externalReferenceCriteria.getLike() != null) { if (!this.conventionService.isNullOrEmpty(path)) return path;
// if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().equals("")) { String completedPath = path;
// completedPath = completedPath.replaceAll("\\{like}", "*");
// completedPath = completedPath.replaceAll("\\{query}", "*"); if (externalReferenceCriteria.getLike() != null) {
// } else { if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().isEmpty()) {
// if (completedPath.contains("{query}")) { completedPath = completedPath.replaceAll("\\{like}", "*");
// completedPath = completedPath.replaceAll("\\{query}", this.calculateQuery(externalReferenceCriteria, queries)); completedPath = completedPath.replaceAll("\\{query}", "*");
// } else { } else {
// completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike()); if (completedPath.contains("{query}")) {
// } completedPath = completedPath.replaceAll("\\{query}", this.replaceLookupFieldQuery(externalReferenceCriteria, queries));
// } } else {
// } else { completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike());
// completedPath = completedPath.replace("{like}", ""); }
// } }
// if (externalReferenceCriteria.getFunderId() != null) { } else {
// String funderPrefix = externalReferenceCriteria.getFunderId().split(":")[0]; completedPath = completedPath.replace("{like}", "");
// String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", ""); }
// if (funderId.toCharArray()[0] == ':') { if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getFunderId())) {
// funderId = externalReferenceCriteria.getFunderId(); String[] founderParts = externalReferenceCriteria.getFunderId().split(":");
// } String funderPrefix = founderParts.length > 0 ? founderParts[0] : "";
// /* String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", "");
// try { funderId = URLEncoder.encode(funderId, "UTF-8"); } catch if (!funderId.isEmpty() && funderId.toCharArray()[0] == ':') {
// (UnsupportedEncodingException e) { logger.error(e.getMessage(), e); } funderId = externalReferenceCriteria.getFunderId();
// */ }
// completedPath = completedPath.replace("{funderId}", funderId); completedPath = completedPath.replace("{funderId}", funderId);
// } } else if(completedPath.contains("{funderId}")){
// else if(completedPath.contains("{funderId}")){ logger.warn("FunderId is null.");
// logger.warn("FunderId is null."); completedPath = completedPath.replace("{funderId}", " ");
// completedPath = completedPath.replace("{funderId}", " "); }
// }
// if (externalReferenceCriteria.getPage() != null) { if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPage())) completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage());
// completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage()); else if (!this.conventionService.isNullOrEmpty(firstPage)) completedPath = completedPath.replace("{page}", firstPage);
// } else { else completedPath = completedPath.replace("{page}", "1");
// if (firstPage != null) {
// completedPath = completedPath.replace("{page}", firstPage); completedPath = completedPath.replace("{pageSize}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPageSize()) ? externalReferenceCriteria.getPageSize() : "60");
// } else { completedPath = completedPath.replace("{host}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getHost()) ? externalReferenceCriteria.getHost() : "");
// completedPath = completedPath.replace("{page}", "1"); completedPath = completedPath.replace("{path}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPath()) ? externalReferenceCriteria.getPath() : "");
// }
// }
// if (externalReferenceCriteria.getPageSize() != null) { return completedPath;
// completedPath = completedPath.replace("{pageSize}", externalReferenceCriteria.getPageSize()); }
// } else {
// completedPath = completedPath.replace("{pageSize}", "60"); private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
// } String replacedPath = replaceLookupFields(apiSource.getUrl(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
// if (externalReferenceCriteria.getHost() != null) { String replacedBody = replaceLookupFields(apiSource.getRequestBody(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries());
// completedPath = completedPath.replace("{host}", externalReferenceCriteria.getHost());
// } else { ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth);
// completedPath = completedPath.replace("{host}", ""); if(externalDataResult != null) {
// } if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
// if (externalReferenceCriteria.getPath() != null) { externalDataResult.setResults(externalDataResult.getResults().stream()
// completedPath = completedPath.replace("{path}", externalReferenceCriteria.getPath()); .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
// } else { .collect(Collectors.toList()));
// completedPath = completedPath.replace("{path}", ""); }
// } externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList());
// return completedPath; if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
// }
// Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
// private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalReferenceCriteria externalReferenceCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType, List<QueryConfig> queries, String auth) throws Exception { if (externalDataResult.getResults().size() > maxResults)
// Set<Integer> pages = new HashSet<>(); throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query");
//
// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries); return externalDataResult;
// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries); }
// else {
// ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth); return new ExternalDataResult();
// if(externalRefernceResult != null) { }
// if (filterType != null && filterType.equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) { }
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
// .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase())) protected ExternalDataResult getExternalDataResults(String urlString, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, String requestBody, String auth) {
// .collect(Collectors.toList()));
// } try {
// if (fetchStrategy == FetchStrategy.FIRST) JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); HttpMethod method;
// switch (apiSource.getHttpMethod()) {
// if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set case GET -> method = HttpMethod.GET;
// for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++) case POST -> method =HttpMethod.POST;
// pages.add(i); default -> throw new MyApplicationException("unrecognized type " + apiSource.getHttpMethod());
// }
// Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
// if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults)) ResponseEntity<String> response = this.getWebClient().method(method).uri(urlString).headers(httpHeaders -> {
// throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query"); if (this.conventionService.isNullOrEmpty(apiSource.getContentType())) {
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(apiSource.getContentType())));
// Optional<ExternalRefernceResult> optionalResults = pages.parallelStream() httpHeaders.setContentType(MediaType.valueOf(apiSource.getContentType()));
// .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth)) }
// .filter(Objects::nonNull) if (auth != null) {
// .reduce((result1, result2) -> { httpHeaders.set("Authorization", auth);
// result1.getResults().addAll(result2.getResults()); }
// return result1; }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
// }); if (response == null || !response.getStatusCode().isSameCodeAs(HttpStatus.OK) || !response.hasBody() || response.getBody() == null) return null;
// ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new);
// remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults()); //do here all the parsing
// List<String> responseContentTypeHeader = response.getHeaders().getOrDefault("Content-Type", null);
// return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); String responseContentType = !this.conventionService.isListNullOrEmpty(responseContentTypeHeader) && responseContentTypeHeader.getFirst() != null ? responseContentTypeHeader.getFirst() : "";
// }
// else { if (responseContentType.contains("json") ) {
// return new LinkedList<>(); DocumentContext jsonContext = JsonPath.parse(response.getBody());
// } return this.parseData(jsonContext, apiSource.getResults());
// } } else {
// throw new MyApplicationException("Unsupported response type" + responseContentType);
// private List<Map<String, Object>> getAllResultsFromUrlWithData(String path, final DataUrlConfiguration jsonDataPath, ExternalReferenceCriteria externalReferenceCriteria, String contentType, String firstPage, String requestBody, String requestType, List<QueryConfig> queries) { }
//
// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries); } catch (Exception exception) {
// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries); logger.error(exception.getMessage(), exception);
// }
// try {
// RestTemplate restTemplate = new RestTemplate(); return null;
// HttpHeaders headers = new HttpHeaders(); }
// HttpEntity<JsonNode> entity;
// ResponseEntity<String> response; private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration<ResultFieldsMappingConfiguration> resultsConfigurationEntity) {
// if (contentType != null && !contentType.isEmpty()) { ExternalDataResult result = new ExternalDataResult();
// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult();
// headers.setContentType(MediaType.valueOf(contentType)); List<Map<String, Object>> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath());
// }
// JsonNode jsonBody = new ObjectMapper().readTree(replacedBody); result.setRawData(rawData);
// entity = new HttpEntity<>(jsonBody, headers);
// if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult();
// response = restTemplate.exchange(replacedPath, HttpMethod.valueOf(requestType), entity, String.class);
// if (response.getStatusCode() == HttpStatus.OK) { List<Map<String, String>> parsedData = new ArrayList<>();
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) { for (Map<String, Object> stringObjectMap: rawData){
// DocumentContext jsonContext = JsonPath.parse(response.getBody()); Map<String, String> map = new HashMap<>();
// return jsonContext.read(jsonDataPath.getPath()); for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){
// } String pathValue = field.getResponsePath();
// } if (!pathValue.contains(".")){
// } if (stringObjectMap.containsKey(pathValue)) {
// catch (Exception exception) { map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
// logger.error(exception.getMessage(), exception); }
// } }else {
// if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
// return new LinkedList<>(); String value = null;
// } Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
// if (fieldObj != null){
// if (fieldObj instanceof Map){
// protected ExternalRefernceResult getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType, String auth) { Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
// if(o instanceof String){
// try { value = (String)o;
// //RestTemplate restTemplate = new RestTemplate(new SimpleClientHttpRequestFactory()); }
// //HttpHeaders headers = new HttpHeaders(); else if(o instanceof Integer){
// //HttpEntity<JsonNode> entity; value = String.valueOf(o);
// ResponseEntity<String> response; }
// /* } else if (fieldObj instanceof List) {
// * URL url = new URL(urlString.replaceAll(" ", "%20")); Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
// * if(o instanceof String){
// * HttpURLConnection con = (HttpURLConnection) url.openConnection(); value = (String)o;
// * con.setRequestMethod("GET"); }
// */ else if(o instanceof Integer){
// /* if (contentType != null && !contentType.isEmpty()) { value = String.valueOf(o);
// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); }
// headers.setContentType(MediaType.valueOf(contentType)); }
// } }
// if (auth != null) { if (value != null){
// headers.set("Authorization", auth); map.put(field.getCode(), value);
// }*/ }
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody); }
//// entity = new HttpEntity<>(jsonBody, headers); }
// }
// parsedData.add(map);
// response = this.getWebClient().method(HttpMethod.valueOf(requestType)).uri(urlString).headers(httpHeaders -> { }
// if (contentType != null && !contentType.isEmpty()) { result.setResults(parsedData);
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); return result;
// httpHeaders.setContentType(MediaType.valueOf(contentType)); }
// }
// if (auth != null) { private static String normalizeValue(Object value) {
// httpHeaders.set("Authorization", auth); if (value instanceof JSONArray jsonArray) {
// }
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) {
// //response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class); return jsonArray.getFirst().toString();
// if (response.getStatusCode() == HttpStatus.OK) { // success } else {
// //do here all the parsing for (Object o : jsonArray) {
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(); if ((o instanceof Map) && ((Map<?, ?>) o).containsKey("content")) {
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) { try {
// DocumentContext jsonContext = JsonPath.parse(response.getBody()); return ((Map<String, String>) o).get("content");
// }
// if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) { catch (ClassCastException e){
// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType, auth); if(((Map<?, ?>) o).get("content") instanceof Integer) {
// } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) { return String.valueOf(((Map<?, ?>) o).get("content"));
// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath); }
// } else { return null;
// externalRefernceResult = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath); }
// } }
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue))) }
// .collect(Collectors.toList())); }
// } } else if (value instanceof Map) {
// else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) { String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
// Class<?> aClass = Class.forName(jsonDataPath.getParseClass()); return ((Map<String, String>)value).get(key);
// JAXBContext jaxbContext = JAXBContext.newInstance(aClass); }
// Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); return value != null ? value.toString() : null;
// StringReader stringReader = new StringReader(response.getBody()); }
// Object data = unmarshaller.unmarshal(stringReader);
// Method reader = null; private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) {
// if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) { List<Map<String, String>> internalResults;
// String camelCaseGetter = jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty() ? "get" + jsonDataPath.getParseField().substring(0, 1).toUpperCase() + jsonDataPath.getParseField().substring(1) : ""; try {
// reader = aClass.getMethod(camelCaseGetter); String filePath = Paths.get(path).toUri().toURL().toString();
// } ObjectMapper mapper = new ObjectMapper();
// ObjectMapper objectMapper = new ObjectMapper(); internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
// List<Map<String, String>> values = new ArrayList<>(); return new ExternalDataResult(searchListMap(internalResults, query));
// int max = 1; } catch (Exception e) {
// if (reader != null) { logger.error(e.getMessage(), e);
// Object invokedField = reader.invoke(data); return new ExternalDataResult();
// if (invokedField instanceof Collection) { }
// max = ((Collection) invokedField).size(); }
// }
// } private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
// for (int i = 0; i< max; i++) { List<Map<String, String>> list = new LinkedList<>();
// Object value; for (Map<String, String> map : internalResults)
// if (reader != null) { {
// Object invokedField = reader.invoke(data); if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
// if (invokedField instanceof Collection) { list.add(map);
// value = ((Collection) invokedField).toArray()[i]; }
// } else { if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
// value = invokedField; list.add(map);
// } }
// } else { }
// value = data; return list;
// } }
// Map<String, String> map = objectMapper.convertValue(value, Map.class);
// if (jsonDataPath.getMergedFields() != null && !jsonDataPath.getMergedFields().isEmpty() && jsonDataPath.getMergedFieldName() != null && !jsonDataPath.getMergedFieldName().isEmpty()) { private String parseBodyString(String bodyString) {
// Map<String, String> finalMap = new HashMap<>(); String finalBodyString = bodyString;
// for (Map.Entry<String, String> entry : map.entrySet()) { if (bodyString.contains("{env:")) {
// if (jsonDataPath.getMergedFields().contains(entry.getKey())) { int index = bodyString.indexOf("{env: ");
// if (!finalMap.containsKey(jsonDataPath.getMergedFieldName())) { while (index >= 0) {
// finalMap.put(jsonDataPath.getMergedFieldName(), entry.getValue()); int endIndex = bodyString.indexOf("}", index + 6);
// } else { String envName = bodyString.substring(index + 6, endIndex);
// finalMap.put(jsonDataPath.getMergedFieldName(), finalMap.get(jsonDataPath.getMergedFieldName()) + " " + entry.getValue()); finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// } index = bodyString.indexOf("{env: ", index + 6);
// } else { }
// finalMap.put(entry.getKey(), entry.getValue()); }
// } return finalBodyString;
// } }
// values.add(finalMap);
// } else { }
// values.add(map);
// }
// }
// externalRefernceResult = new ExternalRefernceResult(values, new HashMap<>(1, 1));
// }
//
// if (externalRefernceResult.getPagination().isEmpty()) {
// externalRefernceResult.getPagination().put("pages", 1);
// externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size());
// }
// return externalRefernceResult;
// }
// } catch (Exception exception) {
// logger.error(exception.getMessage(), exception);
// } //maybe print smth...
//
// return null;
// }
//
// private List<Map<String, String>> getAllResultsFromMockUpJson(String path, String query) {
// List<Map<String, String>> internalResults;
// try {
// String filePath = Paths.get(path).toUri().toURL().toString();
// ObjectMapper mapper = new ObjectMapper();
// internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
// return searchListMap(internalResults, query);
// } catch (Exception e) {
// logger.error(e.getMessage(), e);
// return new LinkedList<>();
// }
// }
//
// private List<Map<String, String>> searchListMap(List<Map<String, String>> internalResults, String query) {
// List<Map<String, String>> list = new LinkedList<>();
// for (Map<String, String> map : internalResults)
// {
// if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) {
// list.add(map);
// }
// if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) {
// list.add(map);
// }
// }
// return list;
// }
//
// private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) {
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) {
// if(dataUrlConfiguration.getFieldsUrlConfiguration().getPid() == null)
// return "pid";
// else
// return "originalId";
// }
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPid() != null && key.equals("pid")) return "pid";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPidTypeField() != null && key.equals("pidTypeField")) return "pidTypeField";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPath() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getPath().replace("'",""))) return "path";
// if (dataUrlConfiguration.getFieldsUrlConfiguration().getHost() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getHost().replace("'",""))) return "host";
// return null;
// }
//
// private String parseBodyString(String bodyString) {
// String finalBodyString = bodyString;
// if (bodyString.contains("{env:")) {
// int index = bodyString.indexOf("{env: ");
// while (index >= 0) {
// int endIndex = bodyString.indexOf("}", index + 6);
// String envName = bodyString.substring(index + 6, endIndex);
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// index = bodyString.indexOf("{env: ", index + 6);
// }
// }
// return finalBodyString;
// }
//
//}

152
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherUtils.java
View File

@ -1,152 +0,0 @@
//package eu.eudat.service.remotefetcher;
//
//import com.fasterxml.jackson.databind.ObjectMapper;
//import com.jayway.jsonpath.DocumentContext;
//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult;
//import net.minidev.json.JSONArray;
//import org.slf4j.Logger;
//import org.slf4j.LoggerFactory;
//
//import java.lang.reflect.InvocationTargetException;
//import java.lang.reflect.Method;
//import java.util.*;
//import java.util.stream.Collectors;
//
//public class RemoteFetcherUtils {
// private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class);
// private static final ObjectMapper mapper = new ObjectMapper();
//
// public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// return new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
// }
//
// public static ExternalRefernceResult getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcherServiceImpl remoteFetcherService, String requestBody, String requestType, String auth) {
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
//
// List<Map<String, String>> multiResults = externalRefernceResult.getResults().stream().map(result -> {
// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria();
// externalReferenceCriteria.setPath(result.get("path"));
// externalReferenceCriteria.setHost(result.get("host"));
// String replacedPath = remoteFetcherService.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalReferenceCriteria, jsonDataPath.getUrlConfiguration().getFirstpage(), jsonDataPath.getUrlConfiguration().getQueries());
// return remoteFetcherService.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType, auth);
// }).filter(Objects::nonNull).map(externalRefernceResult1 -> externalRefernceResult1.getResults().get(0)).collect(Collectors.toList());
// return new ExternalRefernceResult(multiResults, new HashMap<>(1, 1));
// }
//
// public static ExternalRefernceResult getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath),
// new HashMap<>(1, 1));
// externalRefernceResult.getResults().stream().forEach(entry -> {
// String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
// entry.put("name", name);
// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", ""));
// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", ""));
// });
// return externalRefernceResult;
// }
//
// private static List<Map<String, String>> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) {
// List <Map<String, Object>> rawData = jsonContext.read(jsonDataPath.getPath());
// List<Map<String, String>> parsedData = new ArrayList<>();
// rawData.forEach(stringObjectMap -> {
// parsedData.add(new LinkedHashMap<>());
// Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> {
// String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1);
// Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0);
// try {
// String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration()));
// if (value != null) {
// if (field.getName().equals("pid") || field.getName().equals("pidTypeField")) {
// String pid = null;
// Object pidObj = stringObjectMap.get(value.split("\\.")[0]);
// if(pidObj != null){
// if(pidObj instanceof Map){
// Object o = ((Map<String, Object>) pidObj).get(value.split("\\.")[1]);
// if(o instanceof String){
// pid = (String)o;
// }
// else if(o instanceof Integer){
// pid = String.valueOf(o);
// }
// }
// else if(pidObj instanceof List){
// Object o = ((List<Map<String,?>>) pidObj).get(0).get(value.split("\\.")[1]);
// if(o instanceof String){
// pid = (String)o;
// }
// else if(o instanceof Integer){
// pid = String.valueOf(o);
// }
// }
// }
// if(pid != null) {
// if ((field.getName().equals("pid"))){
// parsedData.get(parsedData.size() - 1).put("pid", pid);
// }
// else{
// parsedData.get(parsedData.size() - 1).put("pidTypeField", pid);
// }
// }
// } else {
// value = value.replace("'", "");
// if (value.contains(".")) {
// String[] parts = value.split("\\.");
// Map<String, Object> tempMap = stringObjectMap;
// for (int i = 0; i < parts.length; i++) {
// if (tempMap.containsKey(parts[i])) {
// if (i + 1 < parts.length) {
// tempMap = (Map<String, Object>) tempMap.get(parts[i]);
// } else {
// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(tempMap.get(parts[i]), (field.getName().equals("types") || field.getName().equals("uri"))));
// }
// }
// }
// } else {
// if (stringObjectMap.containsKey(value)) {
// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri"))));
// }
// }
// }
// }
// } catch (IllegalAccessException | InvocationTargetException e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// });
// });
// return parsedData;
// }
//
// private static String normalizeValue(Object value, boolean jsonString) {
// if (value instanceof JSONArray) {
// if (jsonString) {
// return ((JSONArray)value).toJSONString();
// }
// JSONArray jarr = (JSONArray) value;
// if (jarr.get(0) instanceof String) {
// return jarr.get(0).toString();
// } else {
// for (Object o : jarr) {
// if ((o instanceof Map) && ((Map) o).containsKey("content")) {
// try {
// return ((Map<String, String>) o).get("content");
// }
// catch (ClassCastException e){
// if(((Map<?, ?>) o).get("content") instanceof Integer) {
// return String.valueOf(((Map<?, ?>) o).get("content"));
// }
// return null;
// }
// }
// }
// }
// } else if (value instanceof Map) {
// String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
// return ((Map<String, String>)value).get(key);
// }
// return value != null ? value.toString() : null;
// }
//}

57
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/AuthenticationConfiguration.java
View File

@ -1,57 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
public class AuthenticationConfiguration {
private String authUrl;
private String authMethod = "GET";
private String authTokenPath;
private String authRequestBody;
private String type;
public String getAuthUrl() {
return authUrl;
}
@XmlElement(name = "authUrl")
public void setAuthUrl(String authUrl) {
this.authUrl = authUrl;
}
public String getAuthMethod() {
return authMethod;
}
@XmlElement(name = "authUrlMethod")
public void setAuthMethod(String authMethod) {
this.authMethod = authMethod;
}
public String getAuthTokenPath() {
return authTokenPath;
}
@XmlElement(name = "authTokenJpath")
public void setAuthTokenPath(String authTokenPath) {
this.authTokenPath = authTokenPath;
}
public String getAuthRequestBody() {
return authRequestBody;
}
@XmlElement(name = "authUrlBody")
public void setAuthRequestBody(String authRequestBody) {
this.authRequestBody = authRequestBody;
}
public String getType() {
return type;
}
@XmlElement(name = "authType")
public void setType(String type) {
this.type = type;
}
}

134
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataFieldsUrlConfiguration.java
View File

@ -1,134 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
/**
* Created by ikalyvas on 6/29/2018.
*/
public class DataFieldsUrlConfiguration {
private String id;
private String name;
private String pid;
private String pidTypeField;
private String uri;
private String description;
private String source;
private String count;
private String path;
private String host;
private String types;
private String firstName;
private String lastName;
public String getId() {
return id;
}
@XmlElement(name = "id")
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
@XmlElement(name = "name")
public void setName(String name) {
this.name = name;
}
public String getPid() {
return pid;
}
@XmlElement(name = "pid")
public void setPid(String pid) {
this.pid = pid;
}
public String getPidTypeField() {
return pidTypeField;
}
@XmlElement(name = "pidTypeField")
public void setPidTypeField(String pidTypeField) {
this.pidTypeField = pidTypeField;
}
public String getUri() {
return uri;
}
@XmlElement(name = "uri")
public void setUri(String uri) {
this.uri = uri;
}
public String getDescription() {
return description;
}
@XmlElement(name = "description")
public void setDescription(String description) {
this.description = description;
}
public String getSource() {
return source;
}
@XmlElement(name = "source")
public void setSource(String source) {
this.source = source;
}
public String getCount() {
return count;
}
@XmlElement(name = "count")
public void setCount(String count) {
this.count = count;
}
public String getPath() {
return path;
}
@XmlElement(name = "path")
public void setPath(String path) {
this.path = path;
}
public String getHost() {
return host;
}
@XmlElement(name = "host")
public void setHost(String host) {
this.host = host;
}
@XmlElement(name = "types")
public String getTypes() {
return types;
}
public void setTypes(String types) {
this.types = types;
}
@XmlElement(name = "firstName")
public String getFirstName() {
return firstName;
}
public void setFirstName(String firstName) {
this.firstName = firstName;
}
@XmlElement(name = "lastName")
public String getLastName() {
return lastName;
}
public void setLastName(String lastName) {
this.lastName = lastName;
}
}

82
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataUrlConfiguration.java
View File

@ -1,82 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
/**
* Created by ikalyvas on 6/29/2018.
*/
public class DataUrlConfiguration {
private String path;
private DataFieldsUrlConfiguration fieldsUrlConfiguration;
private UrlConfiguration urlConfiguration;
private String parseClass;
private String parseField;
private List<String> mergedFields;
private String mergedFieldName;
public String getPath() {
return path;
}
@XmlElement(name = "path")
public void setPath(String path) {
this.path = path;
}
public DataFieldsUrlConfiguration getFieldsUrlConfiguration() {
return fieldsUrlConfiguration;
}
@XmlElement(name = "fields")
public void setFieldsUrlConfiguration(DataFieldsUrlConfiguration fieldsUrlConfiguration) {
this.fieldsUrlConfiguration = fieldsUrlConfiguration;
}
public UrlConfiguration getUrlConfiguration() {
return urlConfiguration;
}
@XmlElement(name = "urlConfig")
public void setUrlConfiguration(UrlConfiguration urlConfiguration) {
this.urlConfiguration = urlConfiguration;
}
public String getParseClass() {
return parseClass;
}
@XmlElement(name = "parse-class")
public void setParseClass(String parseClass) {
this.parseClass = parseClass;
}
public String getParseField() {
return parseField;
}
@XmlElement(name = "parse-field")
public void setParseField(String parseField) {
this.parseField = parseField;
}
public List<String> getMergedFields() {
return mergedFields;
}
@XmlElementWrapper(name = "merge-fields")
@XmlElement(name = "field")
public void setMergedFields(List<String> mergedFields) {
this.mergedFields = mergedFields;
}
public String getMergedFieldName() {
return mergedFieldName;
}
@XmlElement(name = "merge-field-name")
public void setMergedFieldName(String mergedFieldName) {
this.mergedFieldName = mergedFieldName;
}
}

173
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java
View File

@ -17,153 +17,8 @@ public class ExternalUrls implements Serializable {
Long maxresults; Long maxresults;
GrantUrls grants;
ProjectUrls projects;
RegistryUrls registries;
RepositoryUrls repositories;
PubRepositoryUrls pubRepositories;
JournalUrls journals;
TaxonomiesUrls taxonomies;
PublicationsUrls publications;
ServiceUrls services;
ResearcherUrls researchers;
OrganisationUrls organisations;
DatasetUrls datasets;
/*TagUrls tags;*/
FunderUrls funders;
LicenseUrls licenses;
ValidationUrls validations;
Map<String, PrefillingConfig> prefillings; Map<String, PrefillingConfig> prefillings;
public RegistryUrls getRegistries() {
return registries;
}
@XmlElement(name = "registries")
public void setRegistries(RegistryUrls registries) {
this.registries = registries;
}
public RepositoryUrls getRepositories() {
return repositories;
}
@XmlElement(name = "repositories")
public void setRepositories(RepositoryUrls repositories) {
this.repositories = repositories;
}
public PubRepositoryUrls getPubRepositories() {
return pubRepositories;
}
public void setPubRepositories(PubRepositoryUrls pubRepositories) {
this.pubRepositories = pubRepositories;
}
public void setJournals(JournalUrls journals) {
this.journals = journals;
}
public JournalUrls getJournals() {
return journals;
}
public JournalUrls getJournal() {
return journals;
}
@XmlElement(name = "journals")
public void setJournal(JournalUrls journal) {
this.journals = journal;
}
public TaxonomiesUrls getTaxonomies() {
return taxonomies;
}
@XmlElement(name = "taxonomies")
public void setTaxonomies(TaxonomiesUrls taxonomies) {
this.taxonomies = taxonomies;
}
public PublicationsUrls getPublications() {
return publications;
}
public void setPublications(PublicationsUrls publications) {
this.publications = publications;
}
public ServiceUrls getServices() {
return services;
}
@XmlElement(name = "services")
public void setServices(ServiceUrls services) {
this.services = services;
}
public ResearcherUrls getResearchers() {
return researchers;
}
@XmlElement(name = "researchers")
public void setResearchers(ResearcherUrls researchers) {
this.researchers = researchers;
}
/*public TagUrls getTags() {
return tags;
}
@XmlElement(name = "tags")
public void setTags(TagUrls tags) {
this.tags = tags;
}*/
public OrganisationUrls getOrganisations() {
return organisations;
}
@XmlElement(name = "organisations")
public void setOrganisations(OrganisationUrls organisations) {
this.organisations = organisations;
}
public GrantUrls getGrants() {
return grants;
}
@XmlElement(name = "grants")
public void setGrants(GrantUrls grants) {
this.grants = grants;
}
public ProjectUrls getProjects() {
return projects;
}
@XmlElement(name = "projects")
public void setProjects(ProjectUrls projects) {
this.projects = projects;
}
public FunderUrls getFunders() {
return funders;
}
public void setFunders(FunderUrls funders) {
this.funders = funders;
}
public Long getMaxresults() { public Long getMaxresults() {
return maxresults; return maxresults;
} }
@ -173,34 +28,6 @@ public class ExternalUrls implements Serializable {
this.maxresults = maxresults; this.maxresults = maxresults;
} }
public DatasetUrls getDatasets() {
return datasets;
}
@XmlElement(name = "datasets")
public void setDatasets(DatasetUrls datasets) {
this.datasets = datasets;
}
public LicenseUrls getLicenses() {
return licenses;
}
@XmlElement(name = "licenses")
public void setLicenses(LicenseUrls licenses) {
this.licenses = licenses;
}
public ValidationUrls getValidations() {
return validations;
}
@XmlElement(name = "validators")
public void setValidations(ValidationUrls validations) {
this.validations = validations;
}
public Map<String, PrefillingConfig> getPrefillings() { public Map<String, PrefillingConfig> getPrefillings() {
return prefillings; return prefillings;
} }

48
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/QueryConfig.java
View File

@ -1,48 +0,0 @@
package eu.eudat.service.remotefetcher.config;
import jakarta.xml.bind.annotation.XmlElement;
public class QueryConfig {
private String condition;
private String separator;
private String value;
private Integer ordinal;
public String getCondition() {
return condition;
}
@XmlElement(name = "condition")
public void setCondition(String condition) {
this.condition = condition;
}
public String getSeparator() {
return separator;
}
@XmlElement(name = "separator")
public void setSeparator(String separator) {
this.separator = separator;
}
public String getValue() {
return value;
}
@XmlElement(name = "value")
public void setValue(String value) {
this.value = value;
}
public Integer getOrdinal() {
return ordinal;
}
@XmlElement(name = "ordinal")
public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal;
}
}

128
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/UrlConfiguration.java
View File

@ -1,34 +1,23 @@
package eu.eudat.service.remotefetcher.config; package eu.eudat.service.remotefetcher.config;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper; import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List; import java.util.List;
public class UrlConfiguration { public class UrlConfiguration implements SourceBaseConfiguration {
private String key; private String key;
private String label; private String label;
private Integer ordinal; private Integer ordinal;
private String url; private ReferenceTypeSourceType type;
private boolean dataInListing;
private DataUrlConfiguration data;
private String type;
private String paginationPath;
private String contentType;
private String funderQuery;
private String firstpage;
private String requestType = "GET";
private String requestBody = "";
private String filterType = "remote";
private AuthenticationConfiguration auth;
private List<QueryConfig> queries;
public String getKey() { public String getKey() {
return key; return key;
} }
@XmlElement(name = "key")
public void setKey(String key) { public void setKey(String key) {
this.key = key; this.key = key;
} }
@ -36,121 +25,24 @@ public class UrlConfiguration {
public String getLabel() { public String getLabel() {
return label; return label;
} }
@XmlElement(name = "label")
public void setLabel(String label) { public void setLabel(String label) {
this.label = label; this.label = label;
} }
public String getUrl() {
return url;
}
@XmlElement(name = "url")
public void setUrl(String url) {
this.url = url;
}
public boolean isDataInListing() {
return dataInListing;
}
@XmlElement(name = "dataInListing")
public void setDataInListing(boolean dataInListing) {
this.dataInListing = dataInListing;
}
public Integer getOrdinal() { public Integer getOrdinal() {
return ordinal; return ordinal;
} }
@XmlElement(name = "ordinal")
public void setOrdinal(Integer ordinal) { public void setOrdinal(Integer ordinal) {
this.ordinal = ordinal; this.ordinal = ordinal;
} }
public DataUrlConfiguration getData() { public ReferenceTypeSourceType getType() {
return data;
}
@XmlElement(name = "data")
public void setData(DataUrlConfiguration data) {
this.data = data;
}
public String getPaginationPath() {
return paginationPath;
}
@XmlElement(name = "paginationpath")
public void setPaginationPath(String paginationPath) {
this.paginationPath = paginationPath;
}
public String getType() {
return type; return type;
} }
@XmlElement(name = "type")
public void setType(String type) { public void setType(ReferenceTypeSourceType type) {
this.type = type; this.type = type;
} }
public String getContentType() {
return contentType;
}
@XmlElement(name = "contenttype")
public void setContentType(String contentType) {
this.contentType = contentType;
}
public String getFunderQuery() {
return funderQuery;
}
@XmlElement(name = "funderQuery")
public void setFunderQuery(String funderQuery) {
this.funderQuery = funderQuery;
}
public String getFirstpage() {
return firstpage;
}
@XmlElement(name = "firstPage")
public void setFirstpage(String firstpage) {
this.firstpage = firstpage;
}
public String getRequestType() {
return requestType;
}
@XmlElement(name = "request")
public void setRequestType(String requestType) {
this.requestType = requestType != null ? requestType : "GET";
}
public String getRequestBody() {
return requestBody;
}
@XmlElement(name = "requestBody")
public void setRequestBody(String requestBody) {
this.requestBody = requestBody != null ? requestBody : "";
}
public String getFilterType() {
return filterType;
}
@XmlElement(name = "filterType")
public void setFilterType(String filterType) {
this.filterType = filterType;
}
public List<QueryConfig> getQueries() {
return queries;
}
@XmlElementWrapper
@XmlElement(name = "query")
public void setQueries(List<QueryConfig> queries) {
this.queries = queries;
}
public AuthenticationConfiguration getAuth() {
return auth;
}
@XmlElement(name="authentication")
public void setAuth(AuthenticationConfiguration auth) {
this.auth = auth;
}
} }

18
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/AuthenticationConfiguration.java Normal file
View File

@ -0,0 +1,18 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
public interface AuthenticationConfiguration {
Boolean getEnabled();
String getAuthUrl();
ReferenceTypeExternalApiHTTPMethodType getAuthMethod();
String getAuthTokenPath();
String getAuthRequestBody();
String getType();
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/DatasetUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class DatasetUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/FunderUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class FunderUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

31
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GeneralUrls.java
View File

@ -1,31 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import java.util.ArrayList;
import java.util.List;
public class GeneralUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public GeneralUrls() {
this.urls = new ArrayList<>();
}
@Override
public List<UrlConfiguration> getUrls() {
return urls;
}
@Override
public FetchStrategy getFetchMode() {
return fetchMode;
}
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

5
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java
View File

@ -1,12 +1,13 @@
package eu.eudat.service.remotefetcher.config.entities; package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import java.util.List; import java.util.List;
public abstract class GenericUrls { public abstract class GenericUrls {
public abstract List<UrlConfiguration> getUrls(); public abstract List<SourceBaseConfiguration> getUrls();
public abstract FetchStrategy getFetchMode(); public abstract FetchStrategy getFetchMode();
} }

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GrantUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class GrantUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/JournalUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class JournalUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/LicenseUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class LicenseUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/OrganisationUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class OrganisationUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

4
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java
View File

@ -20,8 +20,8 @@ public class PrefillingSearch extends GenericUrls {
} }
@Override @Override
public List<UrlConfiguration> getUrls() { public List<SourceBaseConfiguration> getUrls() {
List<UrlConfiguration> urls = new ArrayList<>(); List<SourceBaseConfiguration> urls = new ArrayList<>();
urls.add(urlConfig); urls.add(urlConfig);
return urls; return urls;
} }

33
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ProjectUrls.java
View File

@ -1,33 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ProjectUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PubRepositoryUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class PubRepositoryUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PublicationsUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class PublicationsUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

11
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/QueryConfig.java Normal file
View File

@ -0,0 +1,11 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface QueryConfig {
String getCondition();
String getSeparator();
String getValue();
Integer getOrdinal();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RegistryUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class RegistryUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RepositoryUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class RepositoryUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResearcherUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ResearcherUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

6
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultFieldsMappingConfiguration.java Normal file
View File

@ -0,0 +1,6 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface ResultFieldsMappingConfiguration {
String getResponsePath();
String getCode();
}

9
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultsConfiguration.java Normal file
View File

@ -0,0 +1,9 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
public interface ResultsConfiguration <T extends ResultFieldsMappingConfiguration> {
String getResultsArrayPath();
List<T> getFieldsMapping();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ServiceUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ServiceUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

10
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceBaseConfiguration.java Normal file
View File

@ -0,0 +1,10 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
public interface SourceBaseConfiguration {
String getKey();
String getLabel();
Integer getOrdinal();
ReferenceTypeSourceType getType();
}

29
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceExternalApiConfiguration.java Normal file
View File

@ -0,0 +1,29 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType;
import java.util.List;
public interface SourceExternalApiConfiguration<RsConfig extends ResultsConfiguration<? extends ResultFieldsMappingConfiguration>, AuthConfig extends AuthenticationConfiguration, QConfig extends QueryConfig> extends SourceBaseConfiguration {
String getUrl();
RsConfig getResults();
String getPaginationPath();
String getContentType();
String getFirstPage();
ReferenceTypeExternalApiHTTPMethodType getHttpMethod();
String getRequestBody();
String getFilterType();
AuthConfig getAuth();
List<QConfig> getQueries();
}

7
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceStaticOptionConfiguration.java Normal file
View File

@ -0,0 +1,7 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
public interface SourceStaticOptionConfiguration <Option extends StaticOption> extends SourceBaseConfiguration {
List<Option> getOptions();
}

6
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/StaticOption.java Normal file
View File

@ -0,0 +1,6 @@
package eu.eudat.service.remotefetcher.config.entities;
public interface StaticOption {
String getCode();
String getValue();
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/TagUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
/**
* Created by ikalyvas on 7/9/2018.
*/
public class TagUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

34
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/TaxonomiesUrls.java
View File

@ -1,34 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class TaxonomiesUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

35
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ValidationUrls.java
View File

@ -1,35 +0,0 @@
package eu.eudat.service.remotefetcher.config.entities;
import java.util.List;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import jakarta.xml.bind.annotation.XmlElement;
import jakarta.xml.bind.annotation.XmlElementWrapper;
public class ValidationUrls extends GenericUrls {
List<UrlConfiguration> urls;
FetchStrategy fetchMode;
public List<UrlConfiguration> getUrls() {
return urls;
}
@XmlElementWrapper
@XmlElement(name = "urlConfig")
public void setUrls(List<UrlConfiguration> urls) {
this.urls = urls;
}
public FetchStrategy getFetchMode() {
return fetchMode;
}
@XmlElement(name = "fetchMode")
public void setFetchMode(FetchStrategy fetchMode) {
this.fetchMode = fetchMode;
}
}

46
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/models/ExternalDataResult.java Normal file
View File

@ -0,0 +1,46 @@
package eu.eudat.service.remotefetcher.models;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class ExternalDataResult {
List<Map<String, String>> results;
List<Map<String, Object>> rawData;
public ExternalDataResult() {
this.results = new ArrayList<>();
}
public ExternalDataResult(List<Map<String, String>> results) {
this.results = results;
}
public List<Map<String, String>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
this.results = results;
}
public List<Map<String, Object>> getRawData() {
return rawData;
}
public void setRawData(List<Map<String, Object>> rawData) {
this.rawData = rawData;
}
public void addAll(ExternalDataResult other){
if (other == null) return;
if (other.getResults() != null){
if (this.getResults() == null) this.setResults(new ArrayList<>());
this.getResults().addAll(other.getResults());
}
if (other.getRawData() != null){
if (this.getRawData() == null) this.setRawData(new ArrayList<>());
this.getRawData().addAll(other.getRawData());
}
}
}

37
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/models/ExternalRefernceResult.java
View File

@ -1,37 +0,0 @@
package eu.eudat.service.remotefetcher.models;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class ExternalRefernceResult {
List<Map<String, String>> results;
Map<String, Integer> pagination;
public ExternalRefernceResult() {
this.results = new ArrayList<>();
this.pagination = new HashMap<>();
}
public ExternalRefernceResult(List<Map<String, String>> results, Map<String, Integer> pagination) {
this.results = results;
this.pagination = pagination;
}
public List<Map<String, String>> getResults() {
return results;
}
public void setResults(List<Map<String, String>> results) {
this.results = results;
}
public Map<String, Integer> getPagination() {
return pagination;
}
public void setPagination(Map<String, Integer> pagination) {
this.pagination = pagination;
}
}

5
dmp-backend/web/src/main/java/eu/eudat/controllers/v2/ReferenceController.java
View File

@ -11,7 +11,6 @@ import eu.eudat.model.censorship.ReferenceCensor;
import eu.eudat.model.persist.ReferencePersist; import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.model.result.QueryResult; import eu.eudat.model.result.QueryResult;
import eu.eudat.query.ReferenceQuery; import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup;
import eu.eudat.query.lookup.ReferenceLookup; import eu.eudat.query.lookup.ReferenceLookup;
import eu.eudat.query.lookup.ReferenceSearchLookup; import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.reference.ReferenceService; import eu.eudat.service.reference.ReferenceService;
@ -95,12 +94,12 @@ public class ReferenceController {
@PostMapping("search") @PostMapping("search")
public List<Reference> searchReferenceWithDefinition(@RequestBody ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException { public List<Reference> searchReferenceWithDefinition(@RequestBody ReferenceSearchLookup lookup) throws MyNotFoundException {
logger.debug("search with db definition {}", Reference.class.getSimpleName()); logger.debug("search with db definition {}", Reference.class.getSimpleName());
this.censorFactory.censor(ReferenceCensor.class).censor(lookup.getProject(), null); this.censorFactory.censor(ReferenceCensor.class).censor(lookup.getProject(), null);
List<Reference> references = this.referenceService.searchReferenceWithDefinition(lookup); List<Reference> references = this.referenceService.searchReferenceData(lookup);
this.auditService.track(AuditableAction.Reference_Search, "lookup", lookup); this.auditService.track(AuditableAction.Reference_Search, "lookup", lookup);