From 957c1fad58df3155551fe6634c4f812987f4579d Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Mon, 12 Feb 2024 17:43:41 +0200 Subject: [PATCH] reference fetcher changes --- .../AuthenticationConfigurationEntity.java | 3 +- .../referencetype/QueryConfigEntity.java | 3 +- ...enceTypeSourceBaseConfigurationEntity.java | 3 +- ...eSourceExternalApiConfigurationEntity.java | 3 +- ...SourceStaticOptionConfigurationEntity.java | 3 +- .../ReferenceTypeStaticOptionEntity.java | 3 +- ...esultFieldsMappingConfigurationEntity.java | 3 +- .../ResultsConfigurationEntity.java | 3 +- .../DefinitionSearchBuilder.java | 58 -- .../referencesearch/FieldSearchBuilder.java | 88 -- .../ReferenceSearchBuilder.java | 96 -- .../prefilling/PrefillingServiceImpl.java | 10 +- .../service/reference/ReferenceService.java | 6 +- .../reference/ReferenceServiceImpl.java | 735 +++++++------- .../ReferenceTypeServiceImpl.java | 1 - .../remotefetcher/RemoteFetcherService.java | 35 +- .../RemoteFetcherServiceImpl.java | 917 ++++++++---------- .../remotefetcher/RemoteFetcherUtils.java | 152 --- .../config/AuthenticationConfiguration.java | 57 -- .../config/DataFieldsUrlConfiguration.java | 134 --- .../config/DataUrlConfiguration.java | 82 -- .../remotefetcher/config/ExternalUrls.java | 173 ---- .../remotefetcher/config/QueryConfig.java | 48 - .../config/UrlConfiguration.java | 128 +-- .../entities/AuthenticationConfiguration.java | 18 + .../config/entities/DatasetUrls.java | 33 - .../config/entities/FunderUrls.java | 33 - .../config/entities/GeneralUrls.java | 31 - .../config/entities/GenericUrls.java | 5 +- .../config/entities/GrantUrls.java | 35 - .../config/entities/JournalUrls.java | 34 - .../config/entities/LicenseUrls.java | 33 - .../config/entities/OrganisationUrls.java | 34 - .../config/entities/PrefillingSearch.java | 4 +- .../config/entities/ProjectUrls.java | 33 - .../config/entities/PubRepositoryUrls.java | 34 - .../config/entities/PublicationsUrls.java | 34 - .../config/entities/QueryConfig.java | 11 + .../config/entities/RegistryUrls.java | 35 - .../config/entities/RepositoryUrls.java | 34 - .../config/entities/ResearcherUrls.java | 35 - .../ResultFieldsMappingConfiguration.java | 6 + .../config/entities/ResultsConfiguration.java | 9 + .../config/entities/ServiceUrls.java | 35 - .../entities/SourceBaseConfiguration.java | 10 + .../SourceExternalApiConfiguration.java | 29 + .../SourceStaticOptionConfiguration.java | 7 + .../config/entities/StaticOption.java | 6 + .../config/entities/TagUrls.java | 35 - .../config/entities/TaxonomiesUrls.java | 34 - .../config/entities/ValidationUrls.java | 35 - .../models/ExternalDataResult.java | 46 + .../models/ExternalRefernceResult.java | 37 - .../controllers/v2/ReferenceController.java | 5 +- 54 files changed, 923 insertions(+), 2591 deletions(-) delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/DefinitionSearchBuilder.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/FieldSearchBuilder.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/ReferenceSearchBuilder.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherUtils.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/AuthenticationConfiguration.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataFieldsUrlConfiguration.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataUrlConfiguration.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/QueryConfig.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/AuthenticationConfiguration.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/DatasetUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/FunderUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GeneralUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GrantUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/JournalUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/LicenseUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/OrganisationUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ProjectUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PubRepositoryUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PublicationsUrls.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/QueryConfig.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RegistryUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RepositoryUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResearcherUrls.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultFieldsMappingConfiguration.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultsConfiguration.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ServiceUrls.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceBaseConfiguration.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceExternalApiConfiguration.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceStaticOptionConfiguration.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/StaticOption.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/TagUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/TaxonomiesUrls.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ValidationUrls.java create mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/models/ExternalDataResult.java delete mode 100644 dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/models/ExternalRefernceResult.java diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/AuthenticationConfigurationEntity.java b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/AuthenticationConfigurationEntity.java index 3990b2c1f..2af7bf7f4 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/AuthenticationConfigurationEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/AuthenticationConfigurationEntity.java @@ -1,9 +1,10 @@ package eu.eudat.commons.types.referencetype; import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; +import eu.eudat.service.remotefetcher.config.entities.AuthenticationConfiguration; import jakarta.xml.bind.annotation.XmlElement; -public class AuthenticationConfigurationEntity { +public class AuthenticationConfigurationEntity implements AuthenticationConfiguration { private Boolean enabled; private String authUrl; diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/QueryConfigEntity.java b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/QueryConfigEntity.java index 52ab71f86..7edefb26d 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/QueryConfigEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/QueryConfigEntity.java @@ -1,8 +1,9 @@ package eu.eudat.commons.types.referencetype; +import eu.eudat.service.remotefetcher.config.entities.QueryConfig; import jakarta.xml.bind.annotation.XmlElement; -public class QueryConfigEntity { +public class QueryConfigEntity implements QueryConfig { private String condition; private String separator; diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceBaseConfigurationEntity.java b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceBaseConfigurationEntity.java index ead9bec7c..fb10b4981 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceBaseConfigurationEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceBaseConfigurationEntity.java @@ -1,12 +1,13 @@ package eu.eudat.commons.types.referencetype; import eu.eudat.commons.enums.ReferenceTypeSourceType; +import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration; import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElementWrapper; import java.util.List; -public abstract class ReferenceTypeSourceBaseConfigurationEntity { +public abstract class ReferenceTypeSourceBaseConfigurationEntity implements SourceBaseConfiguration { private String key; diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceExternalApiConfigurationEntity.java b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceExternalApiConfigurationEntity.java index 3fbe81027..2905e2b33 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceExternalApiConfigurationEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceExternalApiConfigurationEntity.java @@ -2,11 +2,12 @@ package eu.eudat.commons.types.referencetype; import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; +import eu.eudat.service.remotefetcher.config.entities.SourceExternalApiConfiguration; import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElementWrapper; import java.util.List; -public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity { +public class ReferenceTypeSourceExternalApiConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceExternalApiConfiguration { private String url; private ResultsConfigurationEntity results; diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceStaticOptionConfigurationEntity.java b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceStaticOptionConfigurationEntity.java index 99d6b273d..f8f141068 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceStaticOptionConfigurationEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeSourceStaticOptionConfigurationEntity.java @@ -1,11 +1,12 @@ package eu.eudat.commons.types.referencetype; +import eu.eudat.service.remotefetcher.config.entities.SourceStaticOptionConfiguration; import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElementWrapper; import java.util.List; -public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity { +public class ReferenceTypeSourceStaticOptionConfigurationEntity extends ReferenceTypeSourceBaseConfigurationEntity implements SourceStaticOptionConfiguration { List options; diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeStaticOptionEntity.java b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeStaticOptionEntity.java index e8c4c85bb..f5b85c170 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeStaticOptionEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ReferenceTypeStaticOptionEntity.java @@ -1,8 +1,9 @@ package eu.eudat.commons.types.referencetype; +import eu.eudat.service.remotefetcher.config.entities.StaticOption; import jakarta.xml.bind.annotation.XmlElement; -public class ReferenceTypeStaticOptionEntity { +public class ReferenceTypeStaticOptionEntity implements StaticOption { private String code; diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultFieldsMappingConfigurationEntity.java b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultFieldsMappingConfigurationEntity.java index 28ecf655f..e988a9ee6 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultFieldsMappingConfigurationEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultFieldsMappingConfigurationEntity.java @@ -1,8 +1,9 @@ package eu.eudat.commons.types.referencetype; +import eu.eudat.service.remotefetcher.config.entities.ResultFieldsMappingConfiguration; import jakarta.xml.bind.annotation.XmlElement; -public class ResultFieldsMappingConfigurationEntity { +public class ResultFieldsMappingConfigurationEntity implements ResultFieldsMappingConfiguration { private String code; private String responsePath; diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultsConfigurationEntity.java b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultsConfigurationEntity.java index 84eec895b..7e8071ba6 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultsConfigurationEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/types/referencetype/ResultsConfigurationEntity.java @@ -1,11 +1,12 @@ package eu.eudat.commons.types.referencetype; +import eu.eudat.service.remotefetcher.config.entities.ResultsConfiguration; import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElementWrapper; import java.util.List; -public class ResultsConfigurationEntity { +public class ResultsConfigurationEntity implements ResultsConfiguration { private String resultsArrayPath; private List fieldsMapping; diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/DefinitionSearchBuilder.java b/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/DefinitionSearchBuilder.java deleted file mode 100644 index b66a528d1..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/DefinitionSearchBuilder.java +++ /dev/null @@ -1,58 +0,0 @@ -//package eu.eudat.model.builder.referencesearch; -// -//import eu.eudat.authorization.AuthorizationFlags; -//import eu.eudat.configurations.referencetype.ReferenceTypeProperties; -//import eu.eudat.convention.ConventionService; -//import eu.eudat.model.builder.BaseBuilder; -//import eu.eudat.model.referencedefinition.Definition; -//import gr.cite.tools.data.builder.BuilderFactory; -//import gr.cite.tools.exception.MyApplicationException; -//import gr.cite.tools.fieldset.FieldSet; -//import gr.cite.tools.logging.DataLogEntry; -//import gr.cite.tools.logging.LoggerService; -//import org.slf4j.LoggerFactory; -//import org.springframework.beans.factory.annotation.Autowired; -//import org.springframework.beans.factory.config.ConfigurableBeanFactory; -//import org.springframework.context.annotation.Scope; -//import org.springframework.stereotype.Component; -// -//import java.util.*; -// -//@Component -//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) -//public class DefinitionSearchBuilder extends BaseBuilder> { -// -// private final BuilderFactory builderFactory; -// private final ReferenceTypeProperties properties; -// private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); -// -// @Autowired -// public DefinitionSearchBuilder( -// ConventionService conventionService, BuilderFactory builderFactory, ReferenceTypeProperties properties) { -// super(conventionService, new LoggerService(LoggerFactory.getLogger(DefinitionSearchBuilder.class))); -// this.builderFactory = builderFactory; -// this.properties = properties; -// } -// -// public DefinitionSearchBuilder authorize(EnumSet values) { -// this.authorize = values; -// return this; -// } -// -// @Override -// public List build(FieldSet fields, List> data) throws MyApplicationException { -// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0)); -// this.logger.trace(new DataLogEntry("requested fields", fields)); -// if (data == null) -// return new ArrayList<>(); -// -// List models = new ArrayList<>(); -// //for (Map d : data) { -// Definition m = new Definition(); -// m.setFields(this.builderFactory.builder(FieldSearchBuilder.class).authorize(this.authorize).build(null, data)); -// models.add(m); -// //} -// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0)); -// return models; -// } -//} diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/FieldSearchBuilder.java b/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/FieldSearchBuilder.java deleted file mode 100644 index 5480bcec2..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/FieldSearchBuilder.java +++ /dev/null @@ -1,88 +0,0 @@ -//package eu.eudat.model.builder.referencesearch; -// -//import eu.eudat.authorization.AuthorizationFlags; -//import eu.eudat.commons.enums.ReferenceFieldDataType; -//import eu.eudat.configurations.referencetype.ReferenceTypeField; -//import eu.eudat.configurations.referencetype.ReferenceTypeProperties; -//import eu.eudat.convention.ConventionService; -//import eu.eudat.model.builder.BaseBuilder; -//import eu.eudat.model.referencedefinition.Field; -//import gr.cite.tools.exception.MyApplicationException; -//import gr.cite.tools.fieldset.FieldSet; -//import gr.cite.tools.logging.DataLogEntry; -//import gr.cite.tools.logging.LoggerService; -//import org.slf4j.LoggerFactory; -//import org.springframework.beans.factory.annotation.Autowired; -//import org.springframework.beans.factory.config.ConfigurableBeanFactory; -//import org.springframework.context.annotation.Scope; -//import org.springframework.stereotype.Component; -// -//import java.util.*; -// -//@Component -//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) -//public class FieldSearchBuilder extends BaseBuilder> { -// -// private final ReferenceTypeProperties properties; -// private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); -// -// @Autowired -// public FieldSearchBuilder( -// ConventionService conventionService, ReferenceTypeProperties properties) { -// super(conventionService, new LoggerService(LoggerFactory.getLogger(FieldSearchBuilder.class))); -// this.properties = properties; -// } -// -// public FieldSearchBuilder authorize(EnumSet values) { -// this.authorize = values; -// return this; -// } -// -// @Override -// public List build(FieldSet fields, List< Map> data) throws MyApplicationException { -// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0)); -// this.logger.trace(new DataLogEntry("requested fields", fields)); -// if (data == null) -// return new ArrayList<>(); -// -// List models = new ArrayList<>(); -// for (Map d : data) { -// -// ReferenceType referenceType = ReferenceType.valueOf(d.getOrDefault("referenceType", null)); -// List typeFields = this.getPropertiesFields(referenceType); -// if (typeFields.isEmpty()){ -// return new ArrayList<>(); -// } -// -// for (ReferenceTypeField typeField: typeFields){ -// Field m = new Field(); -// m.setCode(typeField.getCode()); -// m.setDataType(ReferenceFieldDataType.valueOf(typeField.getDataType())); -// m.setValue(d.getOrDefault(typeField.getCode(), null)); -// models.add(m); -// } -// } -// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0)); -// return models; -// } -// -// private List getPropertiesFields(ReferenceType referenceType){ -// switch (referenceType) { -// case Taxonomies: return properties.getTaxonomy().get("fields"); -// case Licenses: return properties.getLicence().get("fields"); -// case Publications: return properties.getPublication().get("fields"); -// case Journals: return properties.getJournal().get("fields"); -// case PubRepositories: return properties.getPubRepository().get("fields"); -// case DataRepositories: return properties.getDataRepository().get("fields"); -// case Registries: return properties.getRegistry().get("fields"); -// case Services: return properties.getService().get("fields"); -// case Organizations: return properties.getOrganisation().get("fields"); -// case Datasets: return properties.getDataset().get("fields"); -// case Funder: return properties.getFunder().get("fields"); -// case Grants: return properties.getGrant().get("fields"); -// case Project: return properties.getProject().get("fields"); -// case Researcher: return properties.getResearcher().get("fields"); -// default: return null; -// } -// } -//} diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/ReferenceSearchBuilder.java b/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/ReferenceSearchBuilder.java deleted file mode 100644 index 734feccd7..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/model/builder/referencesearch/ReferenceSearchBuilder.java +++ /dev/null @@ -1,96 +0,0 @@ -//package eu.eudat.model.builder.referencesearch; -// -//import eu.eudat.authorization.AuthorizationFlags; -//import eu.eudat.commons.enums.ReferenceSourceType; -//import eu.eudat.convention.ConventionService; -//import eu.eudat.model.Reference; -//import eu.eudat.model.builder.BaseBuilder; -//import eu.eudat.model.referencedefinition.Definition; -//import eu.eudat.model.referencedefinition.Field; -//import gr.cite.tools.data.builder.BuilderFactory; -//import gr.cite.tools.exception.MyApplicationException; -//import gr.cite.tools.fieldset.FieldSet; -//import gr.cite.tools.logging.DataLogEntry; -//import gr.cite.tools.logging.LoggerService; -//import org.slf4j.LoggerFactory; -//import org.springframework.beans.factory.annotation.Autowired; -//import org.springframework.beans.factory.config.ConfigurableBeanFactory; -//import org.springframework.context.annotation.Scope; -//import org.springframework.stereotype.Component; -// -//import java.util.*; -// -//@Component -//@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) -//public class ReferenceSearchBuilder extends BaseBuilder> { -// -// private final BuilderFactory builderFactory; -// private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); -// -// @Autowired -// public ReferenceSearchBuilder( -// ConventionService conventionService, -// BuilderFactory builderFactory) { -// super(conventionService, new LoggerService(LoggerFactory.getLogger(ReferenceSearchBuilder.class))); -// this.builderFactory = builderFactory; -// } -// -// public ReferenceSearchBuilder authorize(EnumSet values) { -// this.authorize = values; -// return this; -// } -// -// @Override -// public List build(FieldSet fields, List> data) throws MyApplicationException { -// this.logger.debug("building for {} items requesting {} fields", Optional.ofNullable(data).map(List::size).orElse(0), Optional.ofNullable(fields).map(FieldSet::getFields).map(Set::size).orElse(0)); -// this.logger.trace(new DataLogEntry("requested fields", fields)); -// if (data == null) -// return new ArrayList<>(); -// -// List models = new ArrayList<>(); -// for (Map d : data) { -// Reference m = new Reference(); -// if (d.containsKey("id")) m.setId(UUID.fromString(d.getOrDefault("id", null))); -// m.setLabel(d.getOrDefault("name", null)); -// m.setSourceType(ReferenceSourceType.External); -// String type = d.getOrDefault("referenceType", null); -// if (type != null) m.setType(ReferenceType.valueOf(type)); -// -// Definition definition = this.builderFactory.builder(DefinitionSearchBuilder.class).authorize(this.authorize).build(null, d); -// m.setDefinition(definition); -// -// switch (m.getType()){ -// case Researcher: -// case Organizations: -// case Funder: -// case Grants: -// case DataRepositories: -// case PubRepositories: -// case Journals: -// case Datasets: -// case Registries: -// case Services:{ -// for (Field field : definition.getFields()) { -// if (field.getCode().equals("pid") && field.getValue() != null) { -// if (d.containsKey("key")) { -// m.setReference(d.get("key") + ":" + field.getValue()); -// } -// } -// if (field.getCode().equals("tag") && field.getValue() != null){ -// m.setSource(field.getValue()); -// } -// } -// } -// } -// -// m.setDescription(d.getOrDefault("description", null)); -// m.setAbbreviation(d.getOrDefault("abbreviation", null)); -// -// models.add(m); -// -// } -// this.logger.debug("build {} items", Optional.of(models).map(List::size).orElse(0)); -// return models; -// } -// -//} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java index 9c7d89303..f8e57c96b 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java @@ -20,6 +20,7 @@ import eu.eudat.model.descriptionproperties.PropertyDefinitionFieldSetItem; import eu.eudat.model.persist.DescriptionProfilingRequest; import eu.eudat.model.persist.DescriptionProfilingWithDataRequest; import eu.eudat.service.remotefetcher.ExternalUrlConfigProvider; +import eu.eudat.service.remotefetcher.RemoteFetcherService; import eu.eudat.service.remotefetcher.config.entities.*; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; import gr.cite.tools.data.builder.BuilderFactory; @@ -66,6 +67,7 @@ public class PrefillingServiceImpl implements PrefillingService { private final ExternalUrlConfigProvider externalUrlConfigProvider; private final XmlHandlingService xmlHandlingService; private final ValidatorFactory validatorFactory; + private final RemoteFetcherService remoteFetcherService; @Autowired public PrefillingServiceImpl( EntityManager entityManager, @@ -75,8 +77,9 @@ public class PrefillingServiceImpl implements PrefillingService { MessageSource messageSource, JsonHandlingService jsonHandlingService, ExternalUrlConfigProvider externalUrlConfigProvider, - XmlHandlingService xmlHandlingService, - ValidatorFactory validatorFactory) { + XmlHandlingService xmlHandlingService, + ValidatorFactory validatorFactory, + RemoteFetcherService remoteFetcherService) { this.entityManager = entityManager; this.builderFactory = builderFactory; this.queryFactory = queryFactory; @@ -86,6 +89,7 @@ public class PrefillingServiceImpl implements PrefillingService { this.externalUrlConfigProvider = externalUrlConfigProvider; this.xmlHandlingService = xmlHandlingService; this.validatorFactory = validatorFactory; + this.remoteFetcherService = remoteFetcherService; } @Override @@ -98,7 +102,7 @@ public class PrefillingServiceImpl implements PrefillingService { List> map; Map prefillingConfigs = this.externalUrlConfigProvider.getExternalUrls().getPrefillings(); // for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {//TODO new reference logic -// map = remoteFetcherService.getExternalGeneric(externalReferenceCriteria, prefillingConfig.getPrefillingSearch()); +// map = remoteFetcherService.getExternalData(prefillingConfig.getPrefillingSearch().getUrls(), externalReferenceCriteria, prefillingConfig.getPrefillingSearch().getFetchMode()); // prefillings.addAll(map.stream().map(submap -> PrefillingEntity.build(submap, this.jsonHandlingService)).toList()); // if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) { // List> mapData = remoteFetcherService.getExternalGenericWithData(externalReferenceCriteria, prefillingConfig.getPrefillingSearch()); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java index 9ec36e969..610259eec 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java @@ -2,10 +2,12 @@ package eu.eudat.service.reference; import com.fasterxml.jackson.core.JsonProcessingException; import eu.eudat.commons.exceptions.HugeResultSetException; +import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.model.Reference; import eu.eudat.model.persist.ReferencePersist; -import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup; import eu.eudat.query.lookup.ReferenceSearchLookup; +import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; +import eu.eudat.service.remotefetcher.models.ExternalDataResult; import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyForbiddenException; import gr.cite.tools.exception.MyNotFoundException; @@ -24,5 +26,5 @@ public interface ReferenceService { void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException; - List searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException; + List searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException; } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java index 790412ecf..f78be7775 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java @@ -1,18 +1,12 @@ package eu.eudat.service.reference; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.jayway.jsonpath.DocumentContext; -import com.jayway.jsonpath.JsonPath; import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.authorization.Permission; import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceFieldDataType; import eu.eudat.commons.enums.ReferenceSourceType; -import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; -import eu.eudat.commons.enums.ReferenceTypeSourceType; -import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.commons.types.reference.DefinitionEntity; import eu.eudat.commons.types.reference.FieldEntity; import eu.eudat.commons.types.referencetype.*; @@ -20,17 +14,18 @@ import eu.eudat.convention.ConventionService; import eu.eudat.data.ReferenceEntity; import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.model.Reference; +import eu.eudat.model.ReferenceType; import eu.eudat.model.builder.ReferenceBuilder; import eu.eudat.model.deleter.ReferenceDeleter; import eu.eudat.model.persist.ReferencePersist; import eu.eudat.model.persist.referencedefinition.DefinitionPersist; import eu.eudat.model.persist.referencedefinition.FieldPersist; import eu.eudat.query.ReferenceQuery; -import eu.eudat.query.ReferenceTypeQuery; -import eu.eudat.query.lookup.ReferenceDefinitionSearchLookup; import eu.eudat.query.lookup.ReferenceSearchLookup; -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.models.ExternalRefernceResult; +import eu.eudat.service.remotefetcher.RemoteFetcherService; +import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration; +import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; +import eu.eudat.service.remotefetcher.models.ExternalDataResult; import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.deleter.DeleterFactory; @@ -45,28 +40,17 @@ import gr.cite.tools.logging.LoggerService; import gr.cite.tools.logging.MapLogEntry; import jakarta.persistence.EntityManager; import jakarta.xml.bind.JAXBException; -import net.minidev.json.JSONArray; import org.jetbrains.annotations.NotNull; import org.slf4j.LoggerFactory; import org.springframework.context.MessageSource; import org.springframework.context.i18n.LocaleContextHolder; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpMethod; -import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; -import org.springframework.http.ResponseEntity; -import org.springframework.http.client.reactive.ReactorClientHttpConnector; -import org.springframework.http.codec.json.Jackson2JsonDecoder; import org.springframework.stereotype.Service; -import org.springframework.web.reactive.function.client.WebClient; -import reactor.netty.http.client.HttpClient; import javax.management.InvalidApplicationException; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import java.time.Instant; import java.util.*; -import java.util.stream.Collectors; @Service public class ReferenceServiceImpl implements ReferenceService { @@ -81,17 +65,16 @@ public class ReferenceServiceImpl implements ReferenceService { private final QueryFactory queryFactory; private final XmlHandlingService xmlHandlingService; - private final WebClient client; - + public final RemoteFetcherService remoteFetcherService; public ReferenceServiceImpl( - EntityManager entityManager, - AuthorizationService authorizationService, - DeleterFactory deleterFactory, - BuilderFactory builderFactory, - ConventionService conventionService, - MessageSource messageSource, - QueryFactory queryFactory, - XmlHandlingService xmlHandlingService) { + EntityManager entityManager, + AuthorizationService authorizationService, + DeleterFactory deleterFactory, + BuilderFactory builderFactory, + ConventionService conventionService, + MessageSource messageSource, + QueryFactory queryFactory, + XmlHandlingService xmlHandlingService, RemoteFetcherService remoteFetcherService) { this.entityManager = entityManager; this.authorizationService = authorizationService; this.deleterFactory = deleterFactory; @@ -100,11 +83,7 @@ public class ReferenceServiceImpl implements ReferenceService { this.messageSource = messageSource; this.queryFactory = queryFactory; this.xmlHandlingService = xmlHandlingService; - this.client = WebClient.builder().codecs(clientCodecConfigurer -> { - clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON)); - clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here??? - } - ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build(); + this.remoteFetcherService = remoteFetcherService; } @Override @@ -178,380 +157,340 @@ public class ReferenceServiceImpl implements ReferenceService { this.deleterFactory.deleter(ReferenceDeleter.class).deleteAndSaveByIds(List.of(id)); } -// public List searchReference(ReferenceType externalType, String query, String type) throws HugeResultSet, MyNotFoundException, InvalidApplicationException { -// ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(query); -// -// List> remoteRepos = this.apiContext.getOperationsContext().getRemoteFetcher().get(externalType, externalUrlCriteria, type); -// -// List list = this.fetchFromDb(externalType, query, type, remoteRepos); -// -// list.addAll(remoteRepos.stream().map(FetcherReference::fromRemoteModel).toList()); -// list = list.stream().filter(x -> x.getName().toLowerCase().contains(query.toLowerCase())).collect(Collectors.toList()); -// list.sort(Comparator.comparing(FetcherReference::getName)); -// return list; -// } + @Override + public List searchReferenceData(ReferenceSearchLookup lookup) throws MyNotFoundException { + int initialOffset = 0; + if (lookup.getPage() != null && !lookup.getPage().isEmpty()){ + initialOffset = lookup.getPage().getOffset(); + lookup.getPage().setOffset(0); + } -// @Override -// public List searchReference(ReferenceSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException { -// int initialOffset = 0; -// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){ -// initialOffset = lookup.getPage().getOffset(); -// lookup.getPage().setOffset(0); -// } -// -// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike()); -// -// List> remoteRepos = remoteFetcherService.getReferences(lookup.getType(), externalReferenceCriteria, lookup.getKey()); -// -// List externalModels = this.builderFactory.builder(ReferenceSearchBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), remoteRepos); -// List models = this.fetchFromDb(lookup); -// -// models.addAll(externalModels); -// -// if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).collect(Collectors.toList()); } -// models.sort(Comparator.comparing(Reference::getLabel)); -// -// if (lookup.getPage() != null && !lookup.getPage().isEmpty()){ -// models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList(); -// } -// -// return models; -// } + ReferenceTypeEntity data = this.entityManager.find(ReferenceTypeEntity.class, lookup.getTypeId()); + if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{lookup.getTypeId(), ReferenceType.class.getSimpleName()}, LocaleContextHolder.getLocale())); + + ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(lookup.getLike()); - private List fetchFromDb(ReferenceSearchLookup lookup){ - ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal); + ExternalDataResult remoteRepos = this.getReferenceData(data, externalReferenceCriteria, lookup.getKey()); + + List externalModels = new ArrayList<>(); + if (remoteRepos != null && this.conventionService.isListNullOrEmpty(remoteRepos.getResults())){ + List referenceEntities = new ArrayList<>(); + for (Map result : remoteRepos.getResults()){ + if (result == null || result.isEmpty()) continue;; + ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data, remoteRepos); + referenceEntities.add(referenceEntity); + } + externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities); + } + + List models = this.fetchReferenceFromDb(lookup); + + for (Reference externalReference : externalModels){ + if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null &&x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference); + } + + if (!this.conventionService.isNullOrEmpty(lookup.getLike())) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(lookup.getLike().toLowerCase())).toList(); } + models.sort(Comparator.comparing(Reference::getLabel)); + + if (lookup.getPage() != null && !lookup.getPage().isEmpty()){ + models = models.stream().skip(initialOffset).limit(lookup.getPage().getSize()).toList(); + } + + return models; + } + + @NotNull + private ReferenceEntity buildReferenceEntityFromExternalData(Map result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) { + ReferenceEntity referenceEntity = new ReferenceEntity(); //TODO new reference logic + referenceEntity.setTypeId(data.getId()); + referenceEntity.setIsActive(IsActive.Active); + referenceEntity.setReference(result.getOrDefault("key", null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("pid", null)); + referenceEntity.setSource(result.getOrDefault("tag", null)); + referenceEntity.setAbbreviation(result.getOrDefault("abbreviation", null)); + referenceEntity.setDescription(result.getOrDefault("description", null)); + referenceEntity.setLabel(result.getOrDefault("name", null)); + referenceEntity.setSourceType(ReferenceSourceType.External); + DefinitionEntity definitionEntity = new DefinitionEntity(); + definitionEntity.setFields(new ArrayList<>()); + for (Map.Entry resultValue : result.entrySet()){ + FieldEntity fieldEntity = new FieldEntity(); + fieldEntity.setCode(resultValue.getKey()); + fieldEntity.setValue(resultValue.getValue()); + fieldEntity.setDataType(ReferenceFieldDataType.Text); + definitionEntity.getFields().add(fieldEntity); + } + referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity)); + return referenceEntity; + } + + private List fetchReferenceFromDb(ReferenceSearchLookup lookup){ + ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId()); List data = query.collectAs(lookup.getProject()); return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data); } -// private List fetchFromDb(ReferenceType externalType, String query, String type, List> remoteRepos) throws InvalidApplicationException { -// List list = new LinkedList<>(); -// switch (externalType) { -// case DataRepositories: -// case PubRepositories: -// case Journals: { -// DataRepositoryCriteria criteria = new DataRepositoryCriteria(); -// if (!query.isEmpty()) criteria.setLike(query); -// criteria.setCreationUserId(this.userScope.getUserId()); -// if (type.equals("")) { -// List dataRepositoryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao().getWithCriteria(criteria)).toList(); -// list = dataRepositoryList.stream().map(item -> new FetcherReference().fromDataRepository(item)).collect(Collectors.toList()); -// } -// } -// case Registries: { -// RegistryCriteria criteria = new RegistryCriteria(); -// if (!query.isEmpty()) criteria.setLike(query); -// criteria.setCreationUserId(this.userScope.getUserId()); -// if (type.equals("")) { -// List registryList = (this.apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao().getWithCriteria(criteria)).toList(); -// list = registryList.stream().map(item -> new FetcherReference().fromRegistry(item)).collect(Collectors.toList()); -// } -// } -// case Services: -// { -// ServiceCriteria criteria = new ServiceCriteria(); -// -// if (!query.isEmpty()) criteria.setLike(query); -// criteria.setCreationUserId(this.userScope.getUserId()); -// -// if (type.equals("")) { -// List serviceList = (this.apiContext.getOperationsContext().getDatabaseRepository().getServiceDao().getWithCriteria(criteria)).toList(); -// list = serviceList.stream().map(item -> new FetcherReference().fromService(item)).collect(Collectors.toList()); -// } -// } -// case Datasets:{ -// -// ExternalDatasetCriteria criteria = apiContext.getOperationsContext().getBuilderFactory().getBuilder(ExternalDatasetCriteriaBuilder.class).like(query).build(); -// -// criteria.setCreationUserId(this.userScope.getUserId()); -// QueryableList items = apiContext.getOperationsContext().getDatabaseRepository().getExternalDatasetDao().getWithCriteria(criteria); -// -// list = items.select(item -> new FetcherReference().fromDataset(item)); -// } -// case Taxonomies: -// case Publications: -// case Licenses: -// break; -// } -// -// return list; -// } - - @Override - public List searchReferenceWithDefinition(ReferenceDefinitionSearchLookup lookup) throws HugeResultSetException, MyNotFoundException, InvalidApplicationException { - - ReferenceTypeQuery query = this.queryFactory.query(ReferenceTypeQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).ids(lookup.getReferenceTypeId()); - List datas = query.collectAs(lookup.getProject()); - if (datas.size() != 1 ){ - return null; + private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) { + ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition()); + if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult(); + + ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).toList(), externalReferenceCriteria, key, null); + for (Map result: results.getResults()) { + result.put("referenceType", referenceType.getName()); } - ReferenceTypeDefinitionEntity definition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, datas.get(0).getDefinition()); - List> remoteRepos = this.getAll(definition.getSources(), lookup); - - return null; - } - - private List> getAll ( List sources, ReferenceDefinitionSearchLookup lookup){ - List> results = new LinkedList<>(); - - if (sources == null || sources.isEmpty()) { - return results; - } - - sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal)); - - List apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList(); - apiSources.forEach(source -> { - try { - String auth = null; - if (source.getAuth()!= null) { - //auth = this.getAuthentication(source.getAuth()); - } - results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth)); - } catch (Exception e) { - logger.error(e.getLocalizedMessage(), e); - } - }); - List staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList(); - staticSources.forEach(source -> { - Map map = new HashMap<>(); - source.getOptions().forEach(option -> { - map.put(option.getCode(), option.getValue()); - map.put("tag", source.getLabel()); - map.put("key", source.getKey()); - }); - results.add(map); - }); - return results; } - private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) { - HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name()); - Map response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl()) - .contentType(MediaType.APPLICATION_JSON) - .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) - .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference>() { - })).block(); +// private List> getAll ( List sources, ReferenceDefinitionSearchLookup lookup){ +// List> results = new LinkedList<>(); +// +// if (sources == null || sources.isEmpty()) { +// return results; +// } +// +// sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal)); +// +// List apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList(); +// apiSources.forEach(source -> { +// try { +// String auth = null; +// if (source.getAuth()!= null) { +// //auth = this.getAuthentication(source.getAuth()); +// } +// results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth)); +// } catch (Exception e) { +// logger.error(e.getLocalizedMessage(), e); +// } +// }); +// List staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList(); +// staticSources.forEach(source -> { +// Map map = new HashMap<>(); +// source.getOptions().forEach(option -> { +// map.put(option.getCode(), option.getValue()); +// map.put("tag", source.getLabel()); +// map.put("key", source.getKey()); +// }); +// results.add(map); +// }); +// +// return results; +// } + +// private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) { +// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name()); +// Map response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl()) +// .contentType(MediaType.APPLICATION_JSON) +// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) +// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference>() { +// })).block(); +// +// +// +// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath()); +// } + +// private String parseBodyString(String bodyString) { +// String finalBodyString = bodyString; +// if (bodyString.contains("{env:")) { +// int index = bodyString.indexOf("{env: "); +// while (index >= 0) { +// int endIndex = bodyString.indexOf("}", index + 6); +// String envName = bodyString.substring(index + 6, endIndex); +// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName)); +// index = bodyString.indexOf("{env: ", index + 6); +// } +// } +// return finalBodyString; +// } + +// private List> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List queries, String auth) throws Exception { +// Set pages = new HashSet<>(); +// +// String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries); +// String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries); +// +// ExternalDataResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth); +// if(externalRefernceResult != null) { +// if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) { +// externalRefernceResult.setResults(externalRefernceResult.getResults().stream() +// .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase())) +// .collect(Collectors.toList())); +// } +// if (fetchStrategy == FetchStrategy.FIRST) +// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList()); +// +// //Long maxResults = configLoader.getExternalUrls().getMaxresults(); +// Long maxResults = Long.valueOf(1000); +// if (externalRefernceResult.getResults().size() > maxResults) +// throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getResults().size() + " results... Please submit a more detailed search query"); +// +// Optional optionalResults = pages.parallelStream() +// .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth)) +// .filter(Objects::nonNull) +// .reduce((result1, result2) -> { +// result1.getResults().addAll(result2.getResults()); +// return result1; +// }); +// ExternalDataResult remainingExternalDataResult = optionalResults.orElseGet(ExternalDataResult::new); +// remainingExternalDataResult.getResults().addAll(externalRefernceResult.getResults()); +// +// return remainingExternalDataResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList()); +// } +// else { +// return new LinkedList<>(); +// } +// } + +// private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List queries){ +// String completedPath = urlPath; +// +// if (urlPath.contains("openaire") || urlPath.contains("orcid") ){ +// if (lookup.getLike() != null) { +// completedPath = completedPath.replace("{like}", lookup.getLike()); +// } else { +// completedPath = completedPath.replace("{like}", "*"); +// } +// } +// +// if (urlPath.contains("{like}")){ +// if (lookup.getLike() != null) { +// completedPath = completedPath.replace("{like}", lookup.getLike()); +// } else { +// completedPath = completedPath.replace("{like}", ""); +// } +// } +// +// if (urlPath.contains("{page}")) { +// if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) { +// completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset())); +// } else if (firstPage != null) { +// completedPath = completedPath.replace("{page}", firstPage); +// } else { +// completedPath = completedPath.replace("{page}", "1"); +// } +// } +// +// if (urlPath.contains("{pageSize}")){ +// if (lookup.getPage() != null && lookup.getPage().getSize() > 0) { +// completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize())); +// } else { +// completedPath = completedPath.replace("{pageSize}", "100"); +// } +// } +// +// +// return completedPath; +// } + +// protected ExternalDataResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) { +// +// try { +// ResponseEntity response; +// JsonNode jsonBody = new ObjectMapper().readTree(requestBody); +// +// response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> { +// if (contentType != null && !contentType.isEmpty()) { +// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); +// httpHeaders.setContentType(MediaType.valueOf(contentType)); +// } +// if (auth != null) { +// httpHeaders.set("Authorization", auth); +// } +// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); +// if (response.getStatusCode() == HttpStatus.OK) { // success +// ExternalDataResult externalDataResult = new ExternalDataResult(); +// if (response.getHeaders().get("Content-Type").get(0).contains("json")) { +// DocumentContext jsonContext = JsonPath.parse(response.getBody()); +// externalDataResult = this.getFromJson(jsonContext, resultsEntity); +// } +// +// return externalDataResult; +// } +// } catch (Exception exception) { +// logger.error(exception.getMessage(), exception); +// } +// +// return null; +// } - - return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath()); - } - - private String parseBodyString(String bodyString) { - String finalBodyString = bodyString; - if (bodyString.contains("{env:")) { - int index = bodyString.indexOf("{env: "); - while (index >= 0) { - int endIndex = bodyString.indexOf("}", index + 6); - String envName = bodyString.substring(index + 6, endIndex); - finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName)); - index = bodyString.indexOf("{env: ", index + 6); - } - } - return finalBodyString; - } - - private List> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List queries, String auth) throws Exception { - Set pages = new HashSet<>(); - - String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries); - String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries); - - ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth); - if(externalRefernceResult != null) { - if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) { - externalRefernceResult.setResults(externalRefernceResult.getResults().stream() - .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase())) - .collect(Collectors.toList())); - } - if (fetchStrategy == FetchStrategy.FIRST) - return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList()); - - if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set - for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++) - pages.add(i); - - //Long maxResults = configLoader.getExternalUrls().getMaxresults(); - Long maxResults = Long.valueOf(1000); - if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults)) - throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query"); - - Optional optionalResults = pages.parallelStream() - .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth)) - .filter(Objects::nonNull) - .reduce((result1, result2) -> { - result1.getResults().addAll(result2.getResults()); - return result1; - }); - ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new); - remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults()); - - return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList()); - } - else { - return new LinkedList<>(); - } - } - - private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List queries){ - String completedPath = urlPath; - - if (urlPath.contains("openaire") || urlPath.contains("orcid") ){ - if (lookup.getLike() != null) { - completedPath = completedPath.replace("{like}", lookup.getLike()); - } else { - completedPath = completedPath.replace("{like}", "*"); - } - } - - if (urlPath.contains("{like}")){ - if (lookup.getLike() != null) { - completedPath = completedPath.replace("{like}", lookup.getLike()); - } else { - completedPath = completedPath.replace("{like}", ""); - } - } - - if (urlPath.contains("{page}")) { - if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) { - completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset())); - } else if (firstPage != null) { - completedPath = completedPath.replace("{page}", firstPage); - } else { - completedPath = completedPath.replace("{page}", "1"); - } - } - - if (urlPath.contains("{pageSize}")){ - if (lookup.getPage() != null && lookup.getPage().getSize() > 0) { - completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize())); - } else { - completedPath = completedPath.replace("{pageSize}", "100"); - } - } +// public static ExternalDataResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { +// return new ExternalDataResult(parseData(jsonContext, resultsEntity)); +// } - return completedPath; - } - - protected ExternalRefernceResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) { - - try { - ResponseEntity response; - JsonNode jsonBody = new ObjectMapper().readTree(requestBody); - - response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> { - if (contentType != null && !contentType.isEmpty()) { - httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); - httpHeaders.setContentType(MediaType.valueOf(contentType)); - } - if (auth != null) { - httpHeaders.set("Authorization", auth); - } - }).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); - if (response.getStatusCode() == HttpStatus.OK) { // success - ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(); - if (response.getHeaders().get("Content-Type").get(0).contains("json")) { - DocumentContext jsonContext = JsonPath.parse(response.getBody()); - externalRefernceResult = this.getFromJson(jsonContext, resultsEntity); - } - - - if (externalRefernceResult.getPagination().size() == 0) { - externalRefernceResult.getPagination().put("pages", 1); - externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size()); - } - return externalRefernceResult; - } - } catch (Exception exception) { - logger.error(exception.getMessage(), exception); - } - - return null; - } - - - public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { - return new ExternalRefernceResult(parseData(jsonContext, resultsEntity), - new HashMap<>(1, 1)); - } - - - private static List> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { - List > rawData = jsonContext.read(resultsEntity.getResultsArrayPath()); - List> parsedData = new ArrayList<>(); - - for (Map stringObjectMap: rawData){ - Map map = new HashMap<>(); - for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){ - String pathValue = field.getResponsePath(); - if (!pathValue.contains(".")){ - if (stringObjectMap.containsKey(pathValue)) { - //map.put(field.getCode(), stringObjectMap.get(pathValue)); - map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue))); - } - }else { - if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){ - String value = null; - Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]); - if (fieldObj != null){ - if (fieldObj instanceof Map){ - Object o = ((Map) fieldObj).get(pathValue.split("\\.")[1]); - if(o instanceof String){ - value = (String)o; - } - else if(o instanceof Integer){ - value = String.valueOf(o); - } - } else if (fieldObj instanceof List) { - Object o = ((List>) fieldObj).get(0).get(pathValue.split("\\.")[1]); - if(o instanceof String){ - value = (String)o; - } - else if(o instanceof Integer){ - value = String.valueOf(o); - } - } - } - if (value != null){ - map.put(field.getCode(), value); - } - } - } - } - parsedData.add(map); - } - return parsedData; - } - - - private static String normalizeValue(Object value) { - if (value instanceof JSONArray) { - JSONArray jarr = (JSONArray) value; - if (jarr.get(0) instanceof String) { - return jarr.get(0).toString(); - } else { - for (Object o : jarr) { - if ((o instanceof Map) && ((Map) o).containsKey("content")) { - try { - return ((Map) o).get("content"); - } - catch (ClassCastException e){ - if(((Map) o).get("content") instanceof Integer) { - return String.valueOf(((Map) o).get("content")); - } - return null; - } - } - } - } - } else if (value instanceof Map) { - String key = ((Map)value).containsKey("$") ? "$" : "content"; - return ((Map)value).get(key); - } - return value != null ? value.toString() : null; - } +// private static List> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { +// List > rawData = jsonContext.read(resultsEntity.getResultsArrayPath()); +// List> parsedData = new ArrayList<>(); +// +// for (Map stringObjectMap: rawData){ +// Map map = new HashMap<>(); +// for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){ +// String pathValue = field.getResponsePath(); +// if (!pathValue.contains(".")){ +// if (stringObjectMap.containsKey(pathValue)) { +// //map.put(field.getCode(), stringObjectMap.get(pathValue)); +// map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue))); +// } +// }else { +// if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){ +// String value = null; +// Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]); +// if (fieldObj != null){ +// if (fieldObj instanceof Map){ +// Object o = ((Map) fieldObj).get(pathValue.split("\\.")[1]); +// if(o instanceof String){ +// value = (String)o; +// } +// else if(o instanceof Integer){ +// value = String.valueOf(o); +// } +// } else if (fieldObj instanceof List) { +// Object o = ((List>) fieldObj).get(0).get(pathValue.split("\\.")[1]); +// if(o instanceof String){ +// value = (String)o; +// } +// else if(o instanceof Integer){ +// value = String.valueOf(o); +// } +// } +// } +// if (value != null){ +// map.put(field.getCode(), value); +// } +// } +// } +// } +// parsedData.add(map); +// } +// return parsedData; +// } +// +// +// private static String normalizeValue(Object value) { +// if (value instanceof JSONArray) { +// JSONArray jarr = (JSONArray) value; +// if (jarr.get(0) instanceof String) { +// return jarr.get(0).toString(); +// } else { +// for (Object o : jarr) { +// if ((o instanceof Map) && ((Map) o).containsKey("content")) { +// try { +// return ((Map) o).get("content"); +// } +// catch (ClassCastException e){ +// if(((Map) o).get("content") instanceof Integer) { +// return String.valueOf(((Map) o).get("content")); +// } +// return null; +// } +// } +// } +// } +// } else if (value instanceof Map) { +// String key = ((Map)value).containsKey("$") ? "$" : "content"; +// return ((Map)value).get(key); +// } +// return value != null ? value.toString() : null; +// } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java index adcdbefa0..f10296d6d 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java @@ -18,7 +18,6 @@ import eu.eudat.service.dmpblueprint.DmpBlueprintServiceImpl; import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.deleter.DeleterFactory; -import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyForbiddenException; import gr.cite.tools.exception.MyNotFoundException; diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java index 4e9164c82..c5aae3ef1 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java @@ -1,21 +1,14 @@ -//package eu.eudat.service.remotefetcher; -// -//import eu.eudat.commons.exceptions.HugeResultSetException; -//import eu.eudat.service.remotefetcher.config.entities.GenericUrls; -//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; -//import gr.cite.tools.exception.MyNotFoundException; -// -//import java.util.List; -//import java.util.Map; -// -//public interface RemoteFetcherService { -// List> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException; -// -// GenericUrls getExternalUrls(ReferenceType referenceType); -// -// Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException; -// -// List> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls); -// -// List> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls); -//} +package eu.eudat.service.remotefetcher; + +import eu.eudat.data.ReferenceTypeEntity; +import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration; +import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; +import eu.eudat.service.remotefetcher.criteria.FetchStrategy; +import eu.eudat.service.remotefetcher.models.ExternalDataResult; + +import java.util.List; + +public interface RemoteFetcherService { + ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy); + Integer countExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key); +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java index bd270a2af..ec47510c7 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java @@ -1,533 +1,388 @@ -//package eu.eudat.service.remotefetcher; -// -//import com.fasterxml.jackson.core.type.TypeReference; -//import com.fasterxml.jackson.databind.JsonNode; -//import com.fasterxml.jackson.databind.ObjectMapper; -//import com.jayway.jsonpath.DocumentContext; -//import com.jayway.jsonpath.JsonPath; -//import eu.eudat.commons.exceptions.HugeResultSetException; -//import eu.eudat.service.remotefetcher.config.AuthenticationConfiguration; -//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration; -//import eu.eudat.service.remotefetcher.config.QueryConfig; -//import eu.eudat.service.remotefetcher.config.UrlConfiguration; -//import eu.eudat.service.remotefetcher.config.entities.GenericUrls; -//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult; -//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; -//import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -//import gr.cite.tools.exception.MyNotFoundException; -//import jakarta.xml.bind.JAXBContext; -//import jakarta.xml.bind.Unmarshaller; -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; -//import org.springframework.beans.factory.annotation.Autowired; -//import org.springframework.core.ParameterizedTypeReference; -//import org.springframework.http.*; -//import org.springframework.http.client.reactive.ReactorClientHttpConnector; -//import org.springframework.http.codec.json.Jackson2JsonDecoder; -//import org.springframework.stereotype.Service; -//import org.springframework.web.client.RestTemplate; -//import org.springframework.web.reactive.function.client.WebClient; -//import reactor.netty.http.client.HttpClient; -// -//import java.io.File; -//import java.io.StringReader; -//import java.lang.reflect.Method; -//import java.nio.file.Paths; -//import java.util.*; -//import java.util.stream.Collectors; -// -//@Service -//public class RemoteFetcherServiceImpl implements RemoteFetcherService { -// private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class); -// -// private WebClient webClient; -// private final ExternalUrlConfigProvider externalUrlConfigProvider; -// @Autowired -// public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider) { -// this.externalUrlConfigProvider = externalUrlConfigProvider; -// } -// -// private WebClient getWebClient(){ -// if (this.webClient == null) { -// this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> { -// clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON)); -// clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here??? -// } -// ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build(); -// } -// return webClient; -// } -// -// @Override -// public List> getReferences(ReferenceType referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException { -// FetchStrategy fetchStrategy = null; -// -//// GenericUrls exGenericUrls = this.getExternalUrls(referenceType); -// List urlConfigs = key != null && !key.isEmpty() ? exGenericUrls.getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) -// : exGenericUrls.getUrls(); -// -// List> results = getAll(urlConfigs, fetchStrategy, externalReferenceCriteria); -// for (Map result: results) { -// result.put("referenceType", referenceType.name()); -// } -// return results; -// } -// -// @Override -// public GenericUrls getExternalUrls(ReferenceType referenceType) { -// return switch (referenceType) { -// case Taxonomies -> this.externalUrlConfigProvider.getExternalUrls().getTaxonomies(); -// case Licenses -> this.externalUrlConfigProvider.getExternalUrls().getLicenses(); -// case Publications -> this.externalUrlConfigProvider.getExternalUrls().getPublications(); -// case Journals -> this.externalUrlConfigProvider.getExternalUrls().getJournals(); -// case PubRepositories -> this.externalUrlConfigProvider.getExternalUrls().getPubRepositories(); -// case DataRepositories -> this.externalUrlConfigProvider.getExternalUrls().getRepositories(); -// case Registries -> this.externalUrlConfigProvider.getExternalUrls().getRegistries(); -// case Services -> this.externalUrlConfigProvider.getExternalUrls().getServices(); -// case Grants -> this.externalUrlConfigProvider.getExternalUrls().getGrants(); -// case Organizations -> this.externalUrlConfigProvider.getExternalUrls().getOrganisations(); -// case Datasets -> this.externalUrlConfigProvider.getExternalUrls().getDatasets(); -// case Funder -> this.externalUrlConfigProvider.getExternalUrls().getFunders(); -// case Project -> this.externalUrlConfigProvider.getExternalUrls().getProjects(); -// case Researcher -> this.externalUrlConfigProvider.getExternalUrls().getResearchers(); -// default -> throw new IllegalArgumentException("Type not found" + referenceType); -// }; -// } -// -// @Override -// public Integer countEntries(ExternalReferenceCriteria externalReferenceCriteria, String key) throws MyNotFoundException, HugeResultSetException { -// List urlConfigs = -// key != null && !key.isEmpty() ? this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls().stream().filter(item -> item.getKey().equals(key)).collect(Collectors.toList()) -// : this.externalUrlConfigProvider.getExternalUrls().getValidations().getUrls(); -// FetchStrategy fetchStrategy = this.externalUrlConfigProvider.getExternalUrls().getValidations().getFetchMode(); -// List> data = this.getAll(urlConfigs, fetchStrategy, externalReferenceCriteria); -// return data.size(); -// } -// -// @Override -// public List> getExternalGeneric(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) { -// List urlConfigurations = genericUrls.getUrls(); -// FetchStrategy fetchStrategy = genericUrls.getFetchMode(); -// return getAll(urlConfigurations, fetchStrategy, externalReferenceCriteria); -// } -// -// @Override -// public List> getExternalGenericWithData(ExternalReferenceCriteria externalReferenceCriteria, GenericUrls genericUrls) { -// List urlConfigurations = genericUrls.getUrls(); -// return getAllWithData(urlConfigurations, externalReferenceCriteria); -// } -// -// private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) { -// -// List> results = new LinkedList<>(); -// -// if (urlConfigs == null || urlConfigs.isEmpty()) { -// return results; -// } -//// throw new MyNotFoundException("No Repository urls found in configuration"); -// -// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal)); -// for (UrlConfiguration urlConfiguration : urlConfigs) { -// applyFunderQuery(urlConfiguration, externalReferenceCriteria); -// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) { -// try { -// String auth = null; -// if (urlConfiguration.getAuth() != null) { -// auth = this.getAuthentication(urlConfiguration.getAuth()); -// } -// results.addAll(getAllResultsFromUrl(urlConfiguration.getUrl(), fetchStrategy, urlConfiguration.getData(), urlConfiguration.getPaginationPath(), externalReferenceCriteria, urlConfiguration.getLabel(), urlConfiguration.getKey(), urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getFilterType(), urlConfiguration.getQueries(), auth)); -// } catch (Exception e) { -// logger.error(e.getLocalizedMessage(), e); -// } -// } else if (urlConfiguration.getType() != null && urlConfiguration.getType().equals("Internal")) { -// results.addAll(getAllResultsFromMockUpJson(urlConfiguration.getUrl(), externalReferenceCriteria.getLike())); +package eu.eudat.service.remotefetcher; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.DocumentContext; +import com.jayway.jsonpath.JsonPath; +import eu.eudat.commons.XmlHandlingService; +import eu.eudat.commons.enums.ReferenceTypeSourceType; +import eu.eudat.commons.exceptions.HugeResultSetException; +import eu.eudat.convention.ConventionService; +import eu.eudat.service.remotefetcher.config.entities.*; +import eu.eudat.service.remotefetcher.models.ExternalDataResult; +import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; +import eu.eudat.service.remotefetcher.criteria.FetchStrategy; +import gr.cite.tools.exception.MyApplicationException; +import net.minidev.json.JSONArray; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.*; +import org.springframework.http.client.reactive.ReactorClientHttpConnector; +import org.springframework.http.codec.json.Jackson2JsonDecoder; +import org.springframework.stereotype.Service; +import org.springframework.web.reactive.function.client.WebClient; +import reactor.netty.http.client.HttpClient; + +import java.io.File; +import java.nio.file.Paths; +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class RemoteFetcherServiceImpl implements RemoteFetcherService { + private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class); + + private WebClient webClient; + private final ExternalUrlConfigProvider externalUrlConfigProvider; + private final ConventionService conventionService; + private final XmlHandlingService xmlHandlingService; + @Autowired + public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) { + this.externalUrlConfigProvider = externalUrlConfigProvider; + this.conventionService = conventionService; + this.xmlHandlingService = xmlHandlingService; + } + + private WebClient getWebClient(){ + if (this.webClient == null) { + this.webClient = WebClient.builder().codecs(clientCodecConfigurer -> { + clientCodecConfigurer.defaultCodecs().jackson2JsonDecoder(new Jackson2JsonDecoder(new ObjectMapper(), MediaType.APPLICATION_JSON)); + clientCodecConfigurer.defaultCodecs().maxInMemorySize(2 * ((int) Math.pow(1024, 3))); //GK: Why here??? + } + ).clientConnector(new ReactorClientHttpConnector(HttpClient.create().followRedirect(true))).build(); + } + return webClient; + } + + + + @Override + public ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) { + List apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList(); + if (!this.conventionService.isNullOrEmpty(key)){ + apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList(); + } + if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult(); + + apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal)); + + return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria); + } + + @Override + public Integer countExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key) { + return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size(); + } + + private ExternalDataResult queryExternalData(List sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) { + + ExternalDataResult results = new ExternalDataResult(); + + if (this.conventionService.isListNullOrEmpty(sources)) return new ExternalDataResult(); + + for (SourceBaseConfiguration source : sources) { + if (source.getType() == null || source.getType().equals(ReferenceTypeSourceType.API)) { + try { + SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource = (SourceExternalApiConfiguration)source; + this.applyFunderToQuery(apiSource, externalReferenceCriteria); + + String auth = null; + if (apiSource.getAuth() != null) { + auth = this.buildAuthentication(apiSource.getAuth()); + } + results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth)); + } catch (Exception e) { + logger.error(e.getLocalizedMessage(), e); + } + } else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) { + SourceStaticOptionConfiguration staticSource = (SourceStaticOptionConfiguration)source; + results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO: + } + } + return results; + } + + private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) { + HttpMethod method; + switch (authenticationConfiguration.getAuthMethod()) { + case GET -> method = HttpMethod.GET; + case POST -> method =HttpMethod.POST; + default -> throw new MyApplicationException("unrecognized type " + authenticationConfiguration.getAuthMethod()); + } + + Map response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl()) + .contentType(MediaType.APPLICATION_JSON) + .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) + .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference>() { + })).block(); + if (response == null) throw new MyApplicationException("Authentication " + authenticationConfiguration.getAuthUrl() + " failed"); + + return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null); + } + + + private void applyFunderToQuery(SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) { + //TODO new reference logic +// if (apiSource.getFunderQuery() != null) { +// if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) { +// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery())); +// } +// else { +// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", "")); // } // } -// /* for (UrlConfiguration urlConfig : urlConfigs) { -// ifFunderQueryExist(urlConfig, externalUrlCriteria); -// if (urlConfig.getType() == null || urlConfig.getType().equals("External")) { -// results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage(), urlConfig.getRequestBody(), urlConfig.getRequestType())); -// } else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) { -// results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike())); -// } -// }*/ -// return results; -// } -// -// private String getAuthentication(AuthenticationConfiguration authenticationConfiguration) { -// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod()); -// Map response = this.getWebClient().method(method).uri(authenticationConfiguration.getAuthUrl()) -// .contentType(MediaType.APPLICATION_JSON) -// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) -// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference>() { -// })).block(); -// -// -// -// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath()); -// } -// -// private List> getAllWithData(List urlConfigs, ExternalReferenceCriteria externalReferenceCriteria) { -// -// List> results = new LinkedList<>(); -// -// if (urlConfigs == null || urlConfigs.isEmpty()) { -// return results; -// } -// -// urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal)); -// urlConfigs.forEach(urlConfiguration -> { -// applyFunderQuery(urlConfiguration, externalReferenceCriteria); -// if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) { -// try { -// results.addAll(getAllResultsFromUrlWithData(urlConfiguration.getUrl(), urlConfiguration.getData(), externalReferenceCriteria, urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getQueries())); -// } catch (Exception e) { -// logger.error(e.getLocalizedMessage(), e); -// } -// } -// }); -// return results; -// -// } -// -// private void applyFunderQuery(UrlConfiguration urlConfiguration, ExternalReferenceCriteria externalReferenceCriteria) { -// if (urlConfiguration.getFunderQuery() != null) { -// if (externalReferenceCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) { -// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", urlConfiguration.getFunderQuery())); -// } -// else { -// urlConfiguration.setUrl(urlConfiguration.getUrl().replace("{funderQuery}", "")); -// } -// } -// } -// -// private String calculateQuery(ExternalReferenceCriteria externalReferenceCriteria, List queryConfigs) { -// String finalQuery = ""; -// QueryConfig queryConfig = queryConfigs.stream().filter(queryConfigl -> externalReferenceCriteria.getLike().matches(queryConfigl.getCondition())) -// .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null); -// if (queryConfig != null) { -// if (queryConfig.getSeparator() != null) { -// String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator()); -// finalQuery = queryConfig.getValue(); -// for (int i = 0; i < likes.length; i++) { -// finalQuery = finalQuery.replaceAll("\\{like" + (i+1) + "}", likes[i]); -// } -// } else { -// finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike()); -// } -// -// } -// return finalQuery; -// } -// -// protected String replaceCriteriaOnUrl(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List queries) { -// String completedPath = path; -// if (externalReferenceCriteria.getLike() != null) { -// if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().equals("")) { -// completedPath = completedPath.replaceAll("\\{like}", "*"); -// completedPath = completedPath.replaceAll("\\{query}", "*"); -// } else { -// if (completedPath.contains("{query}")) { -// completedPath = completedPath.replaceAll("\\{query}", this.calculateQuery(externalReferenceCriteria, queries)); -// } else { -// completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike()); -// } -// } -// } else { -// completedPath = completedPath.replace("{like}", ""); -// } -// if (externalReferenceCriteria.getFunderId() != null) { -// String funderPrefix = externalReferenceCriteria.getFunderId().split(":")[0]; -// String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", ""); -// if (funderId.toCharArray()[0] == ':') { -// funderId = externalReferenceCriteria.getFunderId(); -// } -// /* -// try { funderId = URLEncoder.encode(funderId, "UTF-8"); } catch -// (UnsupportedEncodingException e) { logger.error(e.getMessage(), e); } -// */ -// completedPath = completedPath.replace("{funderId}", funderId); -// } -// else if(completedPath.contains("{funderId}")){ -// logger.warn("FunderId is null."); -// completedPath = completedPath.replace("{funderId}", " "); -// } -// if (externalReferenceCriteria.getPage() != null) { -// completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage()); -// } else { -// if (firstPage != null) { -// completedPath = completedPath.replace("{page}", firstPage); -// } else { -// completedPath = completedPath.replace("{page}", "1"); -// } -// } -// if (externalReferenceCriteria.getPageSize() != null) { -// completedPath = completedPath.replace("{pageSize}", externalReferenceCriteria.getPageSize()); -// } else { -// completedPath = completedPath.replace("{pageSize}", "60"); -// } -// if (externalReferenceCriteria.getHost() != null) { -// completedPath = completedPath.replace("{host}", externalReferenceCriteria.getHost()); -// } else { -// completedPath = completedPath.replace("{host}", ""); -// } -// if (externalReferenceCriteria.getPath() != null) { -// completedPath = completedPath.replace("{path}", externalReferenceCriteria.getPath()); -// } else { -// completedPath = completedPath.replace("{path}", ""); -// } -// return completedPath; -// } -// -// private List> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalReferenceCriteria externalReferenceCriteria, String tag, String key, String contentType, String firstPage, String requestBody, String requestType, String filterType, List queries, String auth) throws Exception { -// Set pages = new HashSet<>(); -// -// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries); -// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries); -// -// ExternalRefernceResult externalRefernceResult = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth); -// if(externalRefernceResult != null) { -// if (filterType != null && filterType.equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) { -// externalRefernceResult.setResults(externalRefernceResult.getResults().stream() -// .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase())) -// .collect(Collectors.toList())); -// } -// if (fetchStrategy == FetchStrategy.FIRST) -// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); -// -// if (externalRefernceResult.getPagination() != null && externalRefernceResult.getPagination().get("pages") != null) //if has more pages, add them to the pages set -// for (int i = 2; i <= externalRefernceResult.getPagination().get("pages"); i++) -// pages.add(i); -// -// Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults(); -// if ((maxResults > 0) && (externalRefernceResult.getPagination().get("count") > maxResults)) -// throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalRefernceResult.getPagination().get("count") + " results... Please submit a more detailed search query"); -// -// Optional optionalResults = pages.parallelStream() -// .map(page -> getResultsFromUrl(path + "&page=" + page, jsonDataPath, jsonPaginationPath, contentType, replacedBody, requestType, auth)) -// .filter(Objects::nonNull) -// .reduce((result1, result2) -> { -// result1.getResults().addAll(result2.getResults()); -// return result1; -// }); -// ExternalRefernceResult remainingExternalRefernceResult = optionalResults.orElseGet(ExternalRefernceResult::new); -// remainingExternalRefernceResult.getResults().addAll(externalRefernceResult.getResults()); -// -// return remainingExternalRefernceResult.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); -// } -// else { -// return new LinkedList<>(); -// } -// } -// -// private List> getAllResultsFromUrlWithData(String path, final DataUrlConfiguration jsonDataPath, ExternalReferenceCriteria externalReferenceCriteria, String contentType, String firstPage, String requestBody, String requestType, List queries) { -// -// String replacedPath = replaceCriteriaOnUrl(path, externalReferenceCriteria, firstPage, queries); -// String replacedBody = replaceCriteriaOnUrl(requestBody, externalReferenceCriteria, firstPage, queries); -// -// try { -// RestTemplate restTemplate = new RestTemplate(); -// HttpHeaders headers = new HttpHeaders(); -// HttpEntity entity; -// ResponseEntity response; -// if (contentType != null && !contentType.isEmpty()) { -// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); -// headers.setContentType(MediaType.valueOf(contentType)); -// } -// JsonNode jsonBody = new ObjectMapper().readTree(replacedBody); -// entity = new HttpEntity<>(jsonBody, headers); -// -// response = restTemplate.exchange(replacedPath, HttpMethod.valueOf(requestType), entity, String.class); -// if (response.getStatusCode() == HttpStatus.OK) { -// if (response.getHeaders().get("Content-Type").get(0).contains("json")) { -// DocumentContext jsonContext = JsonPath.parse(response.getBody()); -// return jsonContext.read(jsonDataPath.getPath()); -// } -// } -// } -// catch (Exception exception) { -// logger.error(exception.getMessage(), exception); -// } -// -// return new LinkedList<>(); -// } -// -// -// protected ExternalRefernceResult getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType, String auth) { -// -// try { -// //RestTemplate restTemplate = new RestTemplate(new SimpleClientHttpRequestFactory()); -// //HttpHeaders headers = new HttpHeaders(); -// //HttpEntity entity; -// ResponseEntity response; -// /* -// * URL url = new URL(urlString.replaceAll(" ", "%20")); -// * -// * HttpURLConnection con = (HttpURLConnection) url.openConnection(); -// * con.setRequestMethod("GET"); -// */ -// /* if (contentType != null && !contentType.isEmpty()) { -// headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); -// headers.setContentType(MediaType.valueOf(contentType)); -// } -// if (auth != null) { -// headers.set("Authorization", auth); -// }*/ -// JsonNode jsonBody = new ObjectMapper().readTree(requestBody); -//// entity = new HttpEntity<>(jsonBody, headers); -// -// -// response = this.getWebClient().method(HttpMethod.valueOf(requestType)).uri(urlString).headers(httpHeaders -> { -// if (contentType != null && !contentType.isEmpty()) { -// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); -// httpHeaders.setContentType(MediaType.valueOf(contentType)); -// } -// if (auth != null) { -// httpHeaders.set("Authorization", auth); -// } -// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); -// //response = restTemplate.exchange(urlString, HttpMethod.resolve(requestType), entity, String.class); -// if (response.getStatusCode() == HttpStatus.OK) { // success -// //do here all the parsing -// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(); -// if (response.getHeaders().get("Content-Type").get(0).contains("json")) { -// DocumentContext jsonContext = JsonPath.parse(response.getBody()); -// -// if (jsonDataPath.getFieldsUrlConfiguration().getPath() != null) { -// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithRecursiveFetching(jsonContext, jsonDataPath, this, requestBody, requestType, auth); -// } else if (jsonDataPath.getFieldsUrlConfiguration().getFirstName() != null) { -// externalRefernceResult = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath); -// } else { -// externalRefernceResult = RemoteFetcherUtils.getFromJson(jsonContext, jsonDataPath); -// } -// externalRefernceResult.setResults(externalRefernceResult.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue))) -// .collect(Collectors.toList())); -// } -// else if (response.getHeaders().get("Content-Type").get(0).contains("xml")) { -// Class aClass = Class.forName(jsonDataPath.getParseClass()); -// JAXBContext jaxbContext = JAXBContext.newInstance(aClass); -// Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); -// StringReader stringReader = new StringReader(response.getBody()); -// Object data = unmarshaller.unmarshal(stringReader); -// Method reader = null; -// if (jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty()) { -// String camelCaseGetter = jsonDataPath.getParseField() != null && !jsonDataPath.getParseField().isEmpty() ? "get" + jsonDataPath.getParseField().substring(0, 1).toUpperCase() + jsonDataPath.getParseField().substring(1) : ""; -// reader = aClass.getMethod(camelCaseGetter); -// } -// ObjectMapper objectMapper = new ObjectMapper(); -// List> values = new ArrayList<>(); -// int max = 1; -// if (reader != null) { -// Object invokedField = reader.invoke(data); -// if (invokedField instanceof Collection) { -// max = ((Collection) invokedField).size(); -// } -// } -// for (int i = 0; i< max; i++) { -// Object value; -// if (reader != null) { -// Object invokedField = reader.invoke(data); -// if (invokedField instanceof Collection) { -// value = ((Collection) invokedField).toArray()[i]; -// } else { -// value = invokedField; -// } -// } else { -// value = data; -// } -// Map map = objectMapper.convertValue(value, Map.class); -// if (jsonDataPath.getMergedFields() != null && !jsonDataPath.getMergedFields().isEmpty() && jsonDataPath.getMergedFieldName() != null && !jsonDataPath.getMergedFieldName().isEmpty()) { -// Map finalMap = new HashMap<>(); -// for (Map.Entry entry : map.entrySet()) { -// if (jsonDataPath.getMergedFields().contains(entry.getKey())) { -// if (!finalMap.containsKey(jsonDataPath.getMergedFieldName())) { -// finalMap.put(jsonDataPath.getMergedFieldName(), entry.getValue()); -// } else { -// finalMap.put(jsonDataPath.getMergedFieldName(), finalMap.get(jsonDataPath.getMergedFieldName()) + " " + entry.getValue()); -// } -// } else { -// finalMap.put(entry.getKey(), entry.getValue()); -// } -// } -// values.add(finalMap); -// } else { -// values.add(map); -// } -// } -// externalRefernceResult = new ExternalRefernceResult(values, new HashMap<>(1, 1)); -// } -// -// if (externalRefernceResult.getPagination().isEmpty()) { -// externalRefernceResult.getPagination().put("pages", 1); -// externalRefernceResult.getPagination().put("count", externalRefernceResult.getResults().size()); -// } -// return externalRefernceResult; -// } -// } catch (Exception exception) { -// logger.error(exception.getMessage(), exception); -// } //maybe print smth... -// -// return null; -// } -// -// private List> getAllResultsFromMockUpJson(String path, String query) { -// List> internalResults; -// try { -// String filePath = Paths.get(path).toUri().toURL().toString(); -// ObjectMapper mapper = new ObjectMapper(); -// internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); -// return searchListMap(internalResults, query); -// } catch (Exception e) { -// logger.error(e.getMessage(), e); -// return new LinkedList<>(); -// } -// } -// -// private List> searchListMap(List> internalResults, String query) { -// List> list = new LinkedList<>(); -// for (Map map : internalResults) -// { -// if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) { -// list.add(map); -// } -// if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) { -// list.add(map); -// } -// } -// return list; -// } -// -// private String transformKey(DataUrlConfiguration dataUrlConfiguration, String key) { -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getId() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getId().replace("'",""))) { -// if(dataUrlConfiguration.getFieldsUrlConfiguration().getPid() == null) -// return "pid"; -// else -// return "originalId"; -// } -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPid() != null && key.equals("pid")) return "pid"; -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPidTypeField() != null && key.equals("pidTypeField")) return "pidTypeField"; -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getDescription() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getDescription().replace("'",""))) return "description"; -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getUri() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getUri().replace("'",""))) return "uri"; -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getName() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getName().replace("'",""))) return "name"; -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getSource() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getSource().replace("'",""))) return "source"; -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getCount() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getCount().replace("'",""))) return "count"; -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getPath() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getPath().replace("'",""))) return "path"; -// if (dataUrlConfiguration.getFieldsUrlConfiguration().getHost() != null && key.equals(dataUrlConfiguration.getFieldsUrlConfiguration().getHost().replace("'",""))) return "host"; -// return null; -// } -// -// private String parseBodyString(String bodyString) { -// String finalBodyString = bodyString; -// if (bodyString.contains("{env:")) { -// int index = bodyString.indexOf("{env: "); -// while (index >= 0) { -// int endIndex = bodyString.indexOf("}", index + 6); -// String envName = bodyString.substring(index + 6, endIndex); -// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName)); -// index = bodyString.indexOf("{env: ", index + 6); -// } -// } -// return finalBodyString; -// } -// -//} + } + + private String replaceLookupFieldQuery(ExternalReferenceCriteria externalReferenceCriteria, List queryConfigs) { + String finalQuery = ""; + if (this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike())) return finalQuery; + QueryConfig queryConfig = queryConfigs.stream().filter(x -> !this.conventionService.isNullOrEmpty(x.getCondition()) && externalReferenceCriteria.getLike().matches(x.getCondition())) + .min((Comparator.comparing(QueryConfig::getOrdinal))).orElse(null); + if (queryConfig != null) { + if (queryConfig.getSeparator() != null) { + String[] likes = externalReferenceCriteria.getLike().split(queryConfig.getSeparator()); + finalQuery = queryConfig.getValue(); + for (int i = 0; i < likes.length; i++) { + finalQuery = finalQuery.replaceAll("\\{like" + (i + 1) + "}", likes[i]); + } + } else { + finalQuery = queryConfig.getValue().replaceAll("\\{like}", externalReferenceCriteria.getLike()); + } + + } + return finalQuery; + } + + protected String replaceLookupFields(String path, ExternalReferenceCriteria externalReferenceCriteria, String firstPage, List queries) { + if (!this.conventionService.isNullOrEmpty(path)) return path; + String completedPath = path; + + if (externalReferenceCriteria.getLike() != null) { + if ((path.contains("openaire") || path.contains("orcid") || path.contains("ror") || path.contains("fairsharing")) && externalReferenceCriteria.getLike().isEmpty()) { + completedPath = completedPath.replaceAll("\\{like}", "*"); + completedPath = completedPath.replaceAll("\\{query}", "*"); + } else { + if (completedPath.contains("{query}")) { + completedPath = completedPath.replaceAll("\\{query}", this.replaceLookupFieldQuery(externalReferenceCriteria, queries)); + } else { + completedPath = completedPath.replaceAll("\\{like}", externalReferenceCriteria.getLike()); + } + } + } else { + completedPath = completedPath.replace("{like}", ""); + } + if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getFunderId())) { + String[] founderParts = externalReferenceCriteria.getFunderId().split(":"); + String funderPrefix = founderParts.length > 0 ? founderParts[0] : ""; + String funderId = externalReferenceCriteria.getFunderId().replace(funderPrefix + ":", ""); + if (!funderId.isEmpty() && funderId.toCharArray()[0] == ':') { + funderId = externalReferenceCriteria.getFunderId(); + } + completedPath = completedPath.replace("{funderId}", funderId); + } else if(completedPath.contains("{funderId}")){ + logger.warn("FunderId is null."); + completedPath = completedPath.replace("{funderId}", " "); + } + + if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPage())) completedPath = completedPath.replace("{page}", externalReferenceCriteria.getPage()); + else if (!this.conventionService.isNullOrEmpty(firstPage)) completedPath = completedPath.replace("{page}", firstPage); + else completedPath = completedPath.replace("{page}", "1"); + + completedPath = completedPath.replace("{pageSize}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPageSize()) ? externalReferenceCriteria.getPageSize() : "60"); + completedPath = completedPath.replace("{host}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getHost()) ? externalReferenceCriteria.getHost() : ""); + completedPath = completedPath.replace("{path}", !this.conventionService.isNullOrEmpty(externalReferenceCriteria.getPath()) ? externalReferenceCriteria.getPath() : ""); + + + return completedPath; + } + + private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception { + String replacedPath = replaceLookupFields(apiSource.getUrl(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries()); + String replacedBody = replaceLookupFields(apiSource.getRequestBody(), externalReferenceCriteria, apiSource.getFirstPage(), apiSource.getQueries()); + + ExternalDataResult externalDataResult = this.getExternalDataResults(replacedPath, apiSource, replacedBody, auth); + if(externalDataResult != null) { + if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) { + externalDataResult.setResults(externalDataResult.getResults().stream() + .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase())) + .collect(Collectors.toList())); + } + externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList()); + if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult; + + Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults(); + if (externalDataResult.getResults().size() > maxResults) + throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query"); + + return externalDataResult; + } + else { + return new ExternalDataResult(); + } + } + + protected ExternalDataResult getExternalDataResults(String urlString, final SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, String requestBody, String auth) { + + try { + JsonNode jsonBody = new ObjectMapper().readTree(requestBody); + HttpMethod method; + switch (apiSource.getHttpMethod()) { + case GET -> method = HttpMethod.GET; + case POST -> method =HttpMethod.POST; + default -> throw new MyApplicationException("unrecognized type " + apiSource.getHttpMethod()); + } + + ResponseEntity response = this.getWebClient().method(method).uri(urlString).headers(httpHeaders -> { + if (this.conventionService.isNullOrEmpty(apiSource.getContentType())) { + httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(apiSource.getContentType()))); + httpHeaders.setContentType(MediaType.valueOf(apiSource.getContentType())); + } + if (auth != null) { + httpHeaders.set("Authorization", auth); + } + }).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); + if (response == null || !response.getStatusCode().isSameCodeAs(HttpStatus.OK) || !response.hasBody() || response.getBody() == null) return null; + + //do here all the parsing + List responseContentTypeHeader = response.getHeaders().getOrDefault("Content-Type", null); + String responseContentType = !this.conventionService.isListNullOrEmpty(responseContentTypeHeader) && responseContentTypeHeader.getFirst() != null ? responseContentTypeHeader.getFirst() : ""; + + if (responseContentType.contains("json") ) { + DocumentContext jsonContext = JsonPath.parse(response.getBody()); + return this.parseData(jsonContext, apiSource.getResults()); + } else { + throw new MyApplicationException("Unsupported response type" + responseContentType); + } + + } catch (Exception exception) { + logger.error(exception.getMessage(), exception); + } + + return null; + } + + private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration resultsConfigurationEntity) { + ExternalDataResult result = new ExternalDataResult(); + if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult(); + List> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath()); + + result.setRawData(rawData); + + if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult(); + + List> parsedData = new ArrayList<>(); + for (Map stringObjectMap: rawData){ + Map map = new HashMap<>(); + for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){ + String pathValue = field.getResponsePath(); + if (!pathValue.contains(".")){ + if (stringObjectMap.containsKey(pathValue)) { + map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue))); + } + }else { + if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){ + String value = null; + Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]); + if (fieldObj != null){ + if (fieldObj instanceof Map){ + Object o = ((Map) fieldObj).get(pathValue.split("\\.")[1]); + if(o instanceof String){ + value = (String)o; + } + else if(o instanceof Integer){ + value = String.valueOf(o); + } + } else if (fieldObj instanceof List) { + Object o = ((List>) fieldObj).get(0).get(pathValue.split("\\.")[1]); + if(o instanceof String){ + value = (String)o; + } + else if(o instanceof Integer){ + value = String.valueOf(o); + } + } + } + if (value != null){ + map.put(field.getCode(), value); + } + } + } + } + parsedData.add(map); + } + result.setResults(parsedData); + return result; + } + + private static String normalizeValue(Object value) { + if (value instanceof JSONArray jsonArray) { + + if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) { + return jsonArray.getFirst().toString(); + } else { + for (Object o : jsonArray) { + if ((o instanceof Map) && ((Map) o).containsKey("content")) { + try { + return ((Map) o).get("content"); + } + catch (ClassCastException e){ + if(((Map) o).get("content") instanceof Integer) { + return String.valueOf(((Map) o).get("content")); + } + return null; + } + } + } + } + } else if (value instanceof Map) { + String key = ((Map)value).containsKey("$") ? "$" : "content"; + return ((Map)value).get(key); + } + return value != null ? value.toString() : null; + } + + private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) { + List> internalResults; + try { + String filePath = Paths.get(path).toUri().toURL().toString(); + ObjectMapper mapper = new ObjectMapper(); + internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); + return new ExternalDataResult(searchListMap(internalResults, query)); + } catch (Exception e) { + logger.error(e.getMessage(), e); + return new ExternalDataResult(); + } + } + + private List> searchListMap(List> internalResults, String query) { + List> list = new LinkedList<>(); + for (Map map : internalResults) + { + if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) { + list.add(map); + } + if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) { + list.add(map); + } + } + return list; + } + + private String parseBodyString(String bodyString) { + String finalBodyString = bodyString; + if (bodyString.contains("{env:")) { + int index = bodyString.indexOf("{env: "); + while (index >= 0) { + int endIndex = bodyString.indexOf("}", index + 6); + String envName = bodyString.substring(index + 6, endIndex); + finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName)); + index = bodyString.indexOf("{env: ", index + 6); + } + } + return finalBodyString; + } + +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherUtils.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherUtils.java deleted file mode 100644 index 081a1c1f1..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherUtils.java +++ /dev/null @@ -1,152 +0,0 @@ -//package eu.eudat.service.remotefetcher; -// -//import com.fasterxml.jackson.databind.ObjectMapper; -//import com.jayway.jsonpath.DocumentContext; -//import eu.eudat.service.remotefetcher.config.DataUrlConfiguration; -//import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; -//import eu.eudat.service.remotefetcher.models.ExternalRefernceResult; -//import net.minidev.json.JSONArray; -//import org.slf4j.Logger; -//import org.slf4j.LoggerFactory; -// -//import java.lang.reflect.InvocationTargetException; -//import java.lang.reflect.Method; -//import java.util.*; -//import java.util.stream.Collectors; -// -//public class RemoteFetcherUtils { -// private final static Logger logger = LoggerFactory.getLogger(RemoteFetcherUtils.class); -// private static final ObjectMapper mapper = new ObjectMapper(); -// -// public static ExternalRefernceResult getFromJson(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { -// return new ExternalRefernceResult(parseData(jsonContext, jsonDataPath), -// new HashMap<>(1, 1)); -// } -// -// public static ExternalRefernceResult getFromJsonWithRecursiveFetching(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath, RemoteFetcherServiceImpl remoteFetcherService, String requestBody, String requestType, String auth) { -// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath), -// new HashMap<>(1, 1)); -// -// List> multiResults = externalRefernceResult.getResults().stream().map(result -> { -// ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(); -// externalReferenceCriteria.setPath(result.get("path")); -// externalReferenceCriteria.setHost(result.get("host")); -// String replacedPath = remoteFetcherService.replaceCriteriaOnUrl(jsonDataPath.getUrlConfiguration().getUrl(), externalReferenceCriteria, jsonDataPath.getUrlConfiguration().getFirstpage(), jsonDataPath.getUrlConfiguration().getQueries()); -// return remoteFetcherService.getResultsFromUrl(replacedPath, jsonDataPath.getUrlConfiguration().getData(), jsonDataPath.getUrlConfiguration().getData().getPath(), jsonDataPath.getUrlConfiguration().getContentType(), requestBody, requestType, auth); -// }).filter(Objects::nonNull).map(externalRefernceResult1 -> externalRefernceResult1.getResults().get(0)).collect(Collectors.toList()); -// return new ExternalRefernceResult(multiResults, new HashMap<>(1, 1)); -// } -// -// public static ExternalRefernceResult getFromJsonWithFirstAndLastName(DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { -// ExternalRefernceResult externalRefernceResult = new ExternalRefernceResult(parseData(jsonContext, jsonDataPath), -// new HashMap<>(1, 1)); -// externalRefernceResult.getResults().stream().forEach(entry -> { -// String name = entry.get(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")) + " " + entry.get(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", "")); -// entry.put("name", name); -// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getFirstName().replace("'", "")); -// entry.remove(jsonDataPath.getFieldsUrlConfiguration().getLastName().replace("'", "")); -// }); -// return externalRefernceResult; -// } -// -// private static List> parseData (DocumentContext jsonContext, DataUrlConfiguration jsonDataPath) { -// List > rawData = jsonContext.read(jsonDataPath.getPath()); -// List> parsedData = new ArrayList<>(); -// rawData.forEach(stringObjectMap -> { -// parsedData.add(new LinkedHashMap<>()); -// Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredFields()).forEach(field -> { -// String getterMethodName = "get" + field.getName().substring(0, 1).toUpperCase(Locale.ROOT) + field.getName().substring(1); -// Method getterMethod = Arrays.stream(jsonDataPath.getFieldsUrlConfiguration().getClass().getDeclaredMethods()).filter(method -> method.getName().equals(getterMethodName)).collect(Collectors.toList()).get(0); -// try { -// String value = ((String) getterMethod.invoke(jsonDataPath.getFieldsUrlConfiguration())); -// if (value != null) { -// if (field.getName().equals("pid") || field.getName().equals("pidTypeField")) { -// String pid = null; -// Object pidObj = stringObjectMap.get(value.split("\\.")[0]); -// if(pidObj != null){ -// if(pidObj instanceof Map){ -// Object o = ((Map) pidObj).get(value.split("\\.")[1]); -// if(o instanceof String){ -// pid = (String)o; -// } -// else if(o instanceof Integer){ -// pid = String.valueOf(o); -// } -// } -// else if(pidObj instanceof List){ -// Object o = ((List>) pidObj).get(0).get(value.split("\\.")[1]); -// if(o instanceof String){ -// pid = (String)o; -// } -// else if(o instanceof Integer){ -// pid = String.valueOf(o); -// } -// } -// } -// if(pid != null) { -// if ((field.getName().equals("pid"))){ -// parsedData.get(parsedData.size() - 1).put("pid", pid); -// } -// else{ -// parsedData.get(parsedData.size() - 1).put("pidTypeField", pid); -// } -// } -// } else { -// value = value.replace("'", ""); -// if (value.contains(".")) { -// String[] parts = value.split("\\."); -// Map tempMap = stringObjectMap; -// for (int i = 0; i < parts.length; i++) { -// if (tempMap.containsKey(parts[i])) { -// if (i + 1 < parts.length) { -// tempMap = (Map) tempMap.get(parts[i]); -// } else { -// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(tempMap.get(parts[i]), (field.getName().equals("types") || field.getName().equals("uri")))); -// } -// } -// } -// } else { -// if (stringObjectMap.containsKey(value)) { -// parsedData.get(parsedData.size() - 1).put(field.getName().equals("types") ? "tags" : value, normalizeValue(stringObjectMap.get(value), (field.getName().equals("types") || field.getName().equals("uri")))); -// } -// } -// } -// } -// } catch (IllegalAccessException | InvocationTargetException e) { -// logger.error(e.getLocalizedMessage(), e); -// } -// }); -// }); -// return parsedData; -// } -// -// private static String normalizeValue(Object value, boolean jsonString) { -// if (value instanceof JSONArray) { -// if (jsonString) { -// return ((JSONArray)value).toJSONString(); -// } -// JSONArray jarr = (JSONArray) value; -// if (jarr.get(0) instanceof String) { -// return jarr.get(0).toString(); -// } else { -// for (Object o : jarr) { -// if ((o instanceof Map) && ((Map) o).containsKey("content")) { -// try { -// return ((Map) o).get("content"); -// } -// catch (ClassCastException e){ -// if(((Map) o).get("content") instanceof Integer) { -// return String.valueOf(((Map) o).get("content")); -// } -// return null; -// } -// } -// } -// } -// } else if (value instanceof Map) { -// String key = ((Map)value).containsKey("$") ? "$" : "content"; -// return ((Map)value).get(key); -// } -// return value != null ? value.toString() : null; -// } -//} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/AuthenticationConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/AuthenticationConfiguration.java deleted file mode 100644 index 4d18e08b1..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/AuthenticationConfiguration.java +++ /dev/null @@ -1,57 +0,0 @@ -package eu.eudat.service.remotefetcher.config; - -import jakarta.xml.bind.annotation.XmlElement; - -public class AuthenticationConfiguration { - - private String authUrl; - private String authMethod = "GET"; - private String authTokenPath; - private String authRequestBody; - private String type; - - public String getAuthUrl() { - return authUrl; - } - - @XmlElement(name = "authUrl") - public void setAuthUrl(String authUrl) { - this.authUrl = authUrl; - } - - public String getAuthMethod() { - return authMethod; - } - - @XmlElement(name = "authUrlMethod") - public void setAuthMethod(String authMethod) { - this.authMethod = authMethod; - } - - public String getAuthTokenPath() { - return authTokenPath; - } - - @XmlElement(name = "authTokenJpath") - public void setAuthTokenPath(String authTokenPath) { - this.authTokenPath = authTokenPath; - } - - public String getAuthRequestBody() { - return authRequestBody; - } - - @XmlElement(name = "authUrlBody") - public void setAuthRequestBody(String authRequestBody) { - this.authRequestBody = authRequestBody; - } - - public String getType() { - return type; - } - - @XmlElement(name = "authType") - public void setType(String type) { - this.type = type; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataFieldsUrlConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataFieldsUrlConfiguration.java deleted file mode 100644 index 10ab09f80..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataFieldsUrlConfiguration.java +++ /dev/null @@ -1,134 +0,0 @@ -package eu.eudat.service.remotefetcher.config; - -import jakarta.xml.bind.annotation.XmlElement; - -/** - * Created by ikalyvas on 6/29/2018. - */ -public class DataFieldsUrlConfiguration { - private String id; - private String name; - private String pid; - private String pidTypeField; - private String uri; - private String description; - private String source; - private String count; - private String path; - private String host; - private String types; - private String firstName; - private String lastName; - - public String getId() { - return id; - } - - @XmlElement(name = "id") - public void setId(String id) { - this.id = id; - } - - public String getName() { - return name; - } - - @XmlElement(name = "name") - public void setName(String name) { - this.name = name; - } - - public String getPid() { - return pid; - } - - @XmlElement(name = "pid") - public void setPid(String pid) { - this.pid = pid; - } - - public String getPidTypeField() { - return pidTypeField; - } - - @XmlElement(name = "pidTypeField") - public void setPidTypeField(String pidTypeField) { - this.pidTypeField = pidTypeField; - } - - public String getUri() { - return uri; - } - - @XmlElement(name = "uri") - public void setUri(String uri) { - this.uri = uri; - } - - public String getDescription() { - return description; - } - - @XmlElement(name = "description") - public void setDescription(String description) { - this.description = description; - } - - public String getSource() { - return source; - } - - @XmlElement(name = "source") - public void setSource(String source) { - this.source = source; - } - - public String getCount() { - return count; - } - - @XmlElement(name = "count") - public void setCount(String count) { - this.count = count; - } - - public String getPath() { - return path; - } - @XmlElement(name = "path") - public void setPath(String path) { - this.path = path; - } - - public String getHost() { - return host; - } - @XmlElement(name = "host") - public void setHost(String host) { - this.host = host; - } - - @XmlElement(name = "types") - public String getTypes() { - return types; - } - public void setTypes(String types) { - this.types = types; - } - - @XmlElement(name = "firstName") - public String getFirstName() { - return firstName; - } - public void setFirstName(String firstName) { - this.firstName = firstName; - } - - @XmlElement(name = "lastName") - public String getLastName() { - return lastName; - } - public void setLastName(String lastName) { - this.lastName = lastName; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataUrlConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataUrlConfiguration.java deleted file mode 100644 index b4d8d10b0..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/DataUrlConfiguration.java +++ /dev/null @@ -1,82 +0,0 @@ -package eu.eudat.service.remotefetcher.config; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -/** - * Created by ikalyvas on 6/29/2018. - */ -public class DataUrlConfiguration { - private String path; - private DataFieldsUrlConfiguration fieldsUrlConfiguration; - private UrlConfiguration urlConfiguration; - private String parseClass; - private String parseField; - private List mergedFields; - private String mergedFieldName; - - public String getPath() { - return path; - } - - @XmlElement(name = "path") - public void setPath(String path) { - this.path = path; - } - - public DataFieldsUrlConfiguration getFieldsUrlConfiguration() { - return fieldsUrlConfiguration; - } - - @XmlElement(name = "fields") - public void setFieldsUrlConfiguration(DataFieldsUrlConfiguration fieldsUrlConfiguration) { - this.fieldsUrlConfiguration = fieldsUrlConfiguration; - } - - public UrlConfiguration getUrlConfiguration() { - return urlConfiguration; - } - - @XmlElement(name = "urlConfig") - public void setUrlConfiguration(UrlConfiguration urlConfiguration) { - this.urlConfiguration = urlConfiguration; - } - - public String getParseClass() { - return parseClass; - } - - @XmlElement(name = "parse-class") - public void setParseClass(String parseClass) { - this.parseClass = parseClass; - } - - public String getParseField() { - return parseField; - } - - @XmlElement(name = "parse-field") - public void setParseField(String parseField) { - this.parseField = parseField; - } - - public List getMergedFields() { - return mergedFields; - } - - @XmlElementWrapper(name = "merge-fields") - @XmlElement(name = "field") - public void setMergedFields(List mergedFields) { - this.mergedFields = mergedFields; - } - - public String getMergedFieldName() { - return mergedFieldName; - } - - @XmlElement(name = "merge-field-name") - public void setMergedFieldName(String mergedFieldName) { - this.mergedFieldName = mergedFieldName; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java index 669525a75..80e27c9b6 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java @@ -17,153 +17,8 @@ public class ExternalUrls implements Serializable { Long maxresults; - GrantUrls grants; - ProjectUrls projects; - RegistryUrls registries; - RepositoryUrls repositories; - PubRepositoryUrls pubRepositories; - JournalUrls journals; - TaxonomiesUrls taxonomies; - PublicationsUrls publications; - ServiceUrls services; - ResearcherUrls researchers; - OrganisationUrls organisations; - DatasetUrls datasets; - /*TagUrls tags;*/ - FunderUrls funders; - LicenseUrls licenses; - ValidationUrls validations; Map prefillings; - - public RegistryUrls getRegistries() { - return registries; - } - - @XmlElement(name = "registries") - public void setRegistries(RegistryUrls registries) { - this.registries = registries; - } - - - public RepositoryUrls getRepositories() { - return repositories; - } - - @XmlElement(name = "repositories") - public void setRepositories(RepositoryUrls repositories) { - this.repositories = repositories; - } - - public PubRepositoryUrls getPubRepositories() { - return pubRepositories; - } - - public void setPubRepositories(PubRepositoryUrls pubRepositories) { - this.pubRepositories = pubRepositories; - } - - public void setJournals(JournalUrls journals) { - this.journals = journals; - } - - public JournalUrls getJournals() { - return journals; - } - - public JournalUrls getJournal() { - return journals; - } - @XmlElement(name = "journals") - public void setJournal(JournalUrls journal) { - this.journals = journal; - } - - public TaxonomiesUrls getTaxonomies() { - return taxonomies; - } - @XmlElement(name = "taxonomies") - public void setTaxonomies(TaxonomiesUrls taxonomies) { - this.taxonomies = taxonomies; - } - - public PublicationsUrls getPublications() { - return publications; - } - - public void setPublications(PublicationsUrls publications) { - this.publications = publications; - } - - public ServiceUrls getServices() { - return services; - } - - @XmlElement(name = "services") - public void setServices(ServiceUrls services) { - this.services = services; - } - - - public ResearcherUrls getResearchers() { - return researchers; - } - - @XmlElement(name = "researchers") - public void setResearchers(ResearcherUrls researchers) { - this.researchers = researchers; - } - - - /*public TagUrls getTags() { - return tags; - } - - @XmlElement(name = "tags") - public void setTags(TagUrls tags) { - this.tags = tags; - }*/ - - - public OrganisationUrls getOrganisations() { - return organisations; - } - - @XmlElement(name = "organisations") - public void setOrganisations(OrganisationUrls organisations) { - this.organisations = organisations; - } - - - public GrantUrls getGrants() { - return grants; - } - - @XmlElement(name = "grants") - public void setGrants(GrantUrls grants) { - this.grants = grants; - } - - - public ProjectUrls getProjects() { - return projects; - } - - @XmlElement(name = "projects") - public void setProjects(ProjectUrls projects) { - this.projects = projects; - } - - - public FunderUrls getFunders() { - return funders; - } - - public void setFunders(FunderUrls funders) { - this.funders = funders; - } - - public Long getMaxresults() { return maxresults; } @@ -173,34 +28,6 @@ public class ExternalUrls implements Serializable { this.maxresults = maxresults; } - - public DatasetUrls getDatasets() { - return datasets; - } - - @XmlElement(name = "datasets") - public void setDatasets(DatasetUrls datasets) { - this.datasets = datasets; - } - - public LicenseUrls getLicenses() { - return licenses; - } - - @XmlElement(name = "licenses") - public void setLicenses(LicenseUrls licenses) { - this.licenses = licenses; - } - - public ValidationUrls getValidations() { - return validations; - } - - @XmlElement(name = "validators") - public void setValidations(ValidationUrls validations) { - this.validations = validations; - } - public Map getPrefillings() { return prefillings; } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/QueryConfig.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/QueryConfig.java deleted file mode 100644 index c7281d73c..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/QueryConfig.java +++ /dev/null @@ -1,48 +0,0 @@ -package eu.eudat.service.remotefetcher.config; - -import jakarta.xml.bind.annotation.XmlElement; - -public class QueryConfig { - - private String condition; - private String separator; - private String value; - private Integer ordinal; - - - public String getCondition() { - return condition; - } - - @XmlElement(name = "condition") - public void setCondition(String condition) { - this.condition = condition; - } - - public String getSeparator() { - return separator; - } - - @XmlElement(name = "separator") - public void setSeparator(String separator) { - this.separator = separator; - } - - public String getValue() { - return value; - } - - @XmlElement(name = "value") - public void setValue(String value) { - this.value = value; - } - - public Integer getOrdinal() { - return ordinal; - } - - @XmlElement(name = "ordinal") - public void setOrdinal(Integer ordinal) { - this.ordinal = ordinal; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/UrlConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/UrlConfiguration.java index 7211f5956..1682f6aad 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/UrlConfiguration.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/UrlConfiguration.java @@ -1,34 +1,23 @@ package eu.eudat.service.remotefetcher.config; +import eu.eudat.commons.enums.ReferenceTypeSourceType; +import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration; import jakarta.xml.bind.annotation.XmlElement; import jakarta.xml.bind.annotation.XmlElementWrapper; import java.util.List; -public class UrlConfiguration { +public class UrlConfiguration implements SourceBaseConfiguration { private String key; private String label; private Integer ordinal; - private String url; - private boolean dataInListing; - private DataUrlConfiguration data; - private String type; - private String paginationPath; - private String contentType; - private String funderQuery; - private String firstpage; - private String requestType = "GET"; - private String requestBody = ""; - private String filterType = "remote"; - private AuthenticationConfiguration auth; - - private List queries; + private ReferenceTypeSourceType type; public String getKey() { return key; } - @XmlElement(name = "key") + public void setKey(String key) { this.key = key; } @@ -36,121 +25,24 @@ public class UrlConfiguration { public String getLabel() { return label; } - @XmlElement(name = "label") + public void setLabel(String label) { this.label = label; } - public String getUrl() { - return url; - } - @XmlElement(name = "url") - public void setUrl(String url) { - this.url = url; - } - - public boolean isDataInListing() { - return dataInListing; - } - @XmlElement(name = "dataInListing") - public void setDataInListing(boolean dataInListing) { - this.dataInListing = dataInListing; - } - public Integer getOrdinal() { return ordinal; } - @XmlElement(name = "ordinal") + public void setOrdinal(Integer ordinal) { this.ordinal = ordinal; } - public DataUrlConfiguration getData() { - return data; - } - @XmlElement(name = "data") - public void setData(DataUrlConfiguration data) { - this.data = data; - } - - public String getPaginationPath() { - return paginationPath; - } - @XmlElement(name = "paginationpath") - public void setPaginationPath(String paginationPath) { - this.paginationPath = paginationPath; - } - - public String getType() { + public ReferenceTypeSourceType getType() { return type; } - @XmlElement(name = "type") - public void setType(String type) { + + public void setType(ReferenceTypeSourceType type) { this.type = type; } - - public String getContentType() { - return contentType; - } - @XmlElement(name = "contenttype") - public void setContentType(String contentType) { - this.contentType = contentType; - } - - public String getFunderQuery() { - return funderQuery; - } - @XmlElement(name = "funderQuery") - public void setFunderQuery(String funderQuery) { - this.funderQuery = funderQuery; - } - - public String getFirstpage() { - return firstpage; - } - @XmlElement(name = "firstPage") - public void setFirstpage(String firstpage) { - this.firstpage = firstpage; - } - - public String getRequestType() { - return requestType; - } - @XmlElement(name = "request") - public void setRequestType(String requestType) { - this.requestType = requestType != null ? requestType : "GET"; - } - public String getRequestBody() { - return requestBody; - } - @XmlElement(name = "requestBody") - public void setRequestBody(String requestBody) { - this.requestBody = requestBody != null ? requestBody : ""; - } - public String getFilterType() { - return filterType; - } - @XmlElement(name = "filterType") - public void setFilterType(String filterType) { - this.filterType = filterType; - } - - public List getQueries() { - return queries; - } - - @XmlElementWrapper - @XmlElement(name = "query") - public void setQueries(List queries) { - this.queries = queries; - } - - public AuthenticationConfiguration getAuth() { - return auth; - } - - @XmlElement(name="authentication") - public void setAuth(AuthenticationConfiguration auth) { - this.auth = auth; - } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/AuthenticationConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/AuthenticationConfiguration.java new file mode 100644 index 000000000..306a03052 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/AuthenticationConfiguration.java @@ -0,0 +1,18 @@ +package eu.eudat.service.remotefetcher.config.entities; + +import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; + +public interface AuthenticationConfiguration { + + Boolean getEnabled(); + + String getAuthUrl(); + + ReferenceTypeExternalApiHTTPMethodType getAuthMethod(); + + String getAuthTokenPath(); + + String getAuthRequestBody(); + + String getType(); +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/DatasetUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/DatasetUrls.java deleted file mode 100644 index 7bd5310e8..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/DatasetUrls.java +++ /dev/null @@ -1,33 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - - -public class DatasetUrls extends GenericUrls{ - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/FunderUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/FunderUrls.java deleted file mode 100644 index cfc3efb8e..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/FunderUrls.java +++ /dev/null @@ -1,33 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class FunderUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GeneralUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GeneralUrls.java deleted file mode 100644 index cf1094605..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GeneralUrls.java +++ /dev/null @@ -1,31 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import java.util.ArrayList; -import java.util.List; - -public class GeneralUrls extends GenericUrls { - - List urls; - FetchStrategy fetchMode; - - public GeneralUrls() { - this.urls = new ArrayList<>(); - } - - @Override - public List getUrls() { - return urls; - } - - @Override - public FetchStrategy getFetchMode() { - return fetchMode; - } - - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java index a350e9964..7c8d0e13a 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java @@ -1,12 +1,13 @@ package eu.eudat.service.remotefetcher.config.entities; import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; import java.util.List; public abstract class GenericUrls { - public abstract List getUrls(); + public abstract List getUrls(); public abstract FetchStrategy getFetchMode(); } + + diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GrantUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GrantUrls.java deleted file mode 100644 index dff4148e2..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GrantUrls.java +++ /dev/null @@ -1,35 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - - -public class GrantUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - -} \ No newline at end of file diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/JournalUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/JournalUrls.java deleted file mode 100644 index ba2a7d2a1..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/JournalUrls.java +++ /dev/null @@ -1,34 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class JournalUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/LicenseUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/LicenseUrls.java deleted file mode 100644 index 4cce39881..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/LicenseUrls.java +++ /dev/null @@ -1,33 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - - -public class LicenseUrls extends GenericUrls{ - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/OrganisationUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/OrganisationUrls.java deleted file mode 100644 index dbb0e8909..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/OrganisationUrls.java +++ /dev/null @@ -1,34 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class OrganisationUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java index 29f448e8c..74f88052e 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java @@ -20,8 +20,8 @@ public class PrefillingSearch extends GenericUrls { } @Override - public List getUrls() { - List urls = new ArrayList<>(); + public List getUrls() { + List urls = new ArrayList<>(); urls.add(urlConfig); return urls; } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ProjectUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ProjectUrls.java deleted file mode 100644 index 5695fa678..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ProjectUrls.java +++ /dev/null @@ -1,33 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class ProjectUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PubRepositoryUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PubRepositoryUrls.java deleted file mode 100644 index 762cc4c27..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PubRepositoryUrls.java +++ /dev/null @@ -1,34 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class PubRepositoryUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PublicationsUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PublicationsUrls.java deleted file mode 100644 index 7a1c8ceea..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PublicationsUrls.java +++ /dev/null @@ -1,34 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class PublicationsUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/QueryConfig.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/QueryConfig.java new file mode 100644 index 000000000..4d8b7cd4c --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/QueryConfig.java @@ -0,0 +1,11 @@ +package eu.eudat.service.remotefetcher.config.entities; + +public interface QueryConfig { + String getCondition(); + + String getSeparator(); + + String getValue(); + + Integer getOrdinal(); +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RegistryUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RegistryUrls.java deleted file mode 100644 index d27893d83..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RegistryUrls.java +++ /dev/null @@ -1,35 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class RegistryUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RepositoryUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RepositoryUrls.java deleted file mode 100644 index d4b0ffec8..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/RepositoryUrls.java +++ /dev/null @@ -1,34 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class RepositoryUrls extends GenericUrls { - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResearcherUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResearcherUrls.java deleted file mode 100644 index ec78860f5..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResearcherUrls.java +++ /dev/null @@ -1,35 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class ResearcherUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultFieldsMappingConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultFieldsMappingConfiguration.java new file mode 100644 index 000000000..0d57682d5 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultFieldsMappingConfiguration.java @@ -0,0 +1,6 @@ +package eu.eudat.service.remotefetcher.config.entities; + +public interface ResultFieldsMappingConfiguration { + String getResponsePath(); + String getCode(); +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultsConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultsConfiguration.java new file mode 100644 index 000000000..f731d3295 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ResultsConfiguration.java @@ -0,0 +1,9 @@ +package eu.eudat.service.remotefetcher.config.entities; + +import java.util.List; + +public interface ResultsConfiguration { + String getResultsArrayPath(); + + List getFieldsMapping(); +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ServiceUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ServiceUrls.java deleted file mode 100644 index 49513761e..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/ServiceUrls.java +++ /dev/null @@ -1,35 +0,0 @@ -package eu.eudat.service.remotefetcher.config.entities; - -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; -import eu.eudat.service.remotefetcher.config.UrlConfiguration; - -import jakarta.xml.bind.annotation.XmlElement; -import jakarta.xml.bind.annotation.XmlElementWrapper; -import java.util.List; - -public class ServiceUrls extends GenericUrls{ - - List urls; - FetchStrategy fetchMode; - - public List getUrls() { - return urls; - } - - @XmlElementWrapper - @XmlElement(name = "urlConfig") - public void setUrls(List urls) { - this.urls = urls; - } - - public FetchStrategy getFetchMode() { - return fetchMode; - } - - @XmlElement(name = "fetchMode") - public void setFetchMode(FetchStrategy fetchMode) { - this.fetchMode = fetchMode; - } - - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceBaseConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceBaseConfiguration.java new file mode 100644 index 000000000..af3af135f --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceBaseConfiguration.java @@ -0,0 +1,10 @@ +package eu.eudat.service.remotefetcher.config.entities; + +import eu.eudat.commons.enums.ReferenceTypeSourceType; + +public interface SourceBaseConfiguration { + String getKey(); + String getLabel(); + Integer getOrdinal(); + ReferenceTypeSourceType getType(); +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceExternalApiConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceExternalApiConfiguration.java new file mode 100644 index 000000000..14359dd34 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceExternalApiConfiguration.java @@ -0,0 +1,29 @@ +package eu.eudat.service.remotefetcher.config.entities; + +import eu.eudat.commons.enums.ReferenceTypeExternalApiHTTPMethodType; + +import java.util.List; + +public interface SourceExternalApiConfiguration, AuthConfig extends AuthenticationConfiguration, QConfig extends QueryConfig> extends SourceBaseConfiguration { + String getUrl(); + + RsConfig getResults(); + + String getPaginationPath(); + + String getContentType(); + + String getFirstPage(); + + ReferenceTypeExternalApiHTTPMethodType getHttpMethod(); + + String getRequestBody(); + + String getFilterType(); + + AuthConfig getAuth(); + + List getQueries(); +} + + diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceStaticOptionConfiguration.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceStaticOptionConfiguration.java new file mode 100644 index 000000000..d6d72adef --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/SourceStaticOptionConfiguration.java @@ -0,0 +1,7 @@ +package eu.eudat.service.remotefetcher.config.entities; + +import java.util.List; + +public interface SourceStaticOptionConfiguration