diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/exceptions/HugeResultSetException.java b/dmp-backend/core/src/main/java/eu/eudat/commons/exceptions/HugeResultSetException.java deleted file mode 100644 index a1d24ad38..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/exceptions/HugeResultSetException.java +++ /dev/null @@ -1,13 +0,0 @@ -package eu.eudat.commons.exceptions; - -public class HugeResultSetException extends Exception { - - private static final long serialVersionUID = -6961447213733280563L; - - - public HugeResultSetException(String message) { - super(message); - } - - -} diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java index f8e57c96b..21ae669cb 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java @@ -60,36 +60,27 @@ public class PrefillingServiceImpl implements PrefillingService { private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(PrefillingServiceImpl.class)); private final EntityManager entityManager; private final BuilderFactory builderFactory; - private final QueryFactory queryFactory; private final ConventionService conventionService; private final MessageSource messageSource; - private final JsonHandlingService jsonHandlingService; private final ExternalUrlConfigProvider externalUrlConfigProvider; private final XmlHandlingService xmlHandlingService; private final ValidatorFactory validatorFactory; - private final RemoteFetcherService remoteFetcherService; @Autowired public PrefillingServiceImpl( EntityManager entityManager, BuilderFactory builderFactory, - QueryFactory queryFactory, ConventionService conventionService, MessageSource messageSource, - JsonHandlingService jsonHandlingService, ExternalUrlConfigProvider externalUrlConfigProvider, XmlHandlingService xmlHandlingService, - ValidatorFactory validatorFactory, - RemoteFetcherService remoteFetcherService) { + ValidatorFactory validatorFactory) { this.entityManager = entityManager; this.builderFactory = builderFactory; - this.queryFactory = queryFactory; this.conventionService = conventionService; this.messageSource = messageSource; - this.jsonHandlingService = jsonHandlingService; this.externalUrlConfigProvider = externalUrlConfigProvider; this.xmlHandlingService = xmlHandlingService; this.validatorFactory = validatorFactory; - this.remoteFetcherService = remoteFetcherService; } @Override diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java index 610259eec..7ab043e5f 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java @@ -1,13 +1,9 @@ package eu.eudat.service.reference; import com.fasterxml.jackson.core.JsonProcessingException; -import eu.eudat.commons.exceptions.HugeResultSetException; -import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.model.Reference; import eu.eudat.model.persist.ReferencePersist; import eu.eudat.query.lookup.ReferenceSearchLookup; -import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; -import eu.eudat.service.remotefetcher.models.ExternalDataResult; import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyForbiddenException; import gr.cite.tools.exception.MyNotFoundException; diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java index ba1531a90..f7a6fea76 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java @@ -243,7 +243,7 @@ public class ReferenceServiceImpl implements ReferenceService { ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition()); if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult(); - ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).collect(Collectors.toList()), externalReferenceCriteria, key, null); + ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).collect(Collectors.toList()), externalReferenceCriteria, key); for (Map result: results.getResults()) { result.put("referenceType", referenceType.getName()); } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java index c5aae3ef1..033fe084b 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java @@ -1,14 +1,12 @@ package eu.eudat.service.remotefetcher; -import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import eu.eudat.service.remotefetcher.models.ExternalDataResult; import java.util.List; public interface RemoteFetcherService { - ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy); + ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key); Integer countExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key); } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java index c81a1c569..1173378dc 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java @@ -7,7 +7,6 @@ import com.jayway.jsonpath.JsonPath; import com.jayway.jsonpath.PathNotFoundException; import eu.eudat.commons.JsonHandlingService; import eu.eudat.commons.enums.ReferenceTypeSourceType; -import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.convention.ConventionService; import eu.eudat.data.ReferenceEntity; import eu.eudat.model.Reference; @@ -15,7 +14,6 @@ import eu.eudat.model.referencedefinition.Field; import eu.eudat.service.remotefetcher.config.entities.*; import eu.eudat.service.remotefetcher.models.ExternalDataResult; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import gr.cite.tools.exception.MyApplicationException; import net.minidev.json.JSONArray; import org.slf4j.Logger; @@ -37,12 +35,10 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class); private WebClient webClient; - private final ExternalUrlConfigProvider externalUrlConfigProvider; private final ConventionService conventionService; private final JsonHandlingService jsonHandlingService; @Autowired - public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, JsonHandlingService jsonHandlingService) { - this.externalUrlConfigProvider = externalUrlConfigProvider; + public RemoteFetcherServiceImpl(ConventionService conventionService, JsonHandlingService jsonHandlingService) { this.conventionService = conventionService; this.jsonHandlingService = jsonHandlingService; } @@ -61,7 +57,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { @Override - public ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) { + public ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key) { List apiSourcesToUse = sources; if (!this.conventionService.isNullOrEmpty(key)){ apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).toList(); @@ -70,15 +66,15 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal)); - return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria); + return this.queryExternalData(sources, externalReferenceCriteria); } @Override public Integer countExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key) { - return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size(); + return getExternalData(sources, externalReferenceCriteria, key).getResults().size(); } - private ExternalDataResult queryExternalData(List sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) { + private ExternalDataResult queryExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria) { ExternalDataResult results = new ExternalDataResult(); @@ -94,7 +90,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { if (apiSource.getAuth() != null && apiSource.getAuth().getEnabled() != null && apiSource.getAuth().getEnabled()) { auth = this.buildAuthentication(apiSource.getAuth()); } - results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth)); + results.addAll(this.queryExternalData(apiSource, externalReferenceCriteria, auth)); } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); } @@ -141,19 +137,6 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null); } - -// private void applyFunderToQuery(SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) { -// //TODO new reference logic -// if (apiSource.getFunderQuery() != null) { -// if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) { -// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery())); -// } -// else { -// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", "")); -// } -// } -// } - private String replaceLookupFieldQuery(String query, ExternalReferenceCriteria externalReferenceCriteria, List> queryConfigs) { String finalQuery = query; String likeValue = this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike()) ? "" : externalReferenceCriteria.getLike(); @@ -222,7 +205,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { return completedPath; } - private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception { + private ExternalDataResult queryExternalData(final SourceExternalApiConfiguration, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception { String replacedPath = replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria); String replacedBody = replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria); @@ -234,12 +217,6 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { .collect(Collectors.toList())); } externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put(ReferenceEntity.KnownFields.SourceLabel, apiSource.getLabel())).peek(x -> x.put(ReferenceEntity.KnownFields.Key, apiSource.getKey())).toList()); - if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult; - - Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults(); - if (externalDataResult.getResults().size() > maxResults) - throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query"); - return externalDataResult; } else { @@ -300,22 +277,24 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { List> parsedData = new ArrayList<>(); for(Object resultItem : results){ Map map = new HashMap<>(); + boolean isValid = true; for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) { if (this.conventionService.isNullOrEmpty(field.getResponsePath()) || this.conventionService.isNullOrEmpty(field.getCode())) continue; try { Object value = JsonPath.parse(resultItem).read(field.getResponsePath()); map.put(field.getCode(), normalizeJsonValue(value)); }catch (PathNotFoundException e){ -// List nextLevelResults = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath() + field.getResponsePath()); -// for(Object nextLevelResultValue : nextLevelResults) { -// map.put(field.getCode(), normalizeJsonValue(nextLevelResultValue)); -// } logger.debug("Json Path Error: " + e.getMessage() + " on source " + jsonHandlingService.toJsonSafe(resultItem)); - //TODO if value don't exist - continue; + if (ReferenceEntity.KnownFields.ReferenceId.equals(field.getCode())) { + isValid = false; + break; + } } } - parsedData.add(map); + if (this.conventionService.isNullOrEmpty(map.getOrDefault(ReferenceEntity.KnownFields.ReferenceId, null))){ + logger.warn("Invalid reference on source " + jsonHandlingService.toJsonSafe(resultItem)); + } + if (isValid) parsedData.add(map); } result.setResults(parsedData); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java index 80e27c9b6..a65b71b06 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java @@ -15,19 +15,8 @@ public class ExternalUrls implements Serializable { private static final long serialVersionUID = -5076364662014107275L; - Long maxresults; - Map prefillings; - public Long getMaxresults() { - return maxresults; - } - - @XmlElement(name = "maxresults") - public void setMaxresults(Long maxresults) { - this.maxresults = maxresults; - } - public Map getPrefillings() { return prefillings; } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java index 7c8d0e13a..777f4788c 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java @@ -1,13 +1,11 @@ package eu.eudat.service.remotefetcher.config.entities; -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import java.util.List; public abstract class GenericUrls { public abstract List getUrls(); - public abstract FetchStrategy getFetchMode(); } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java index 74f88052e..badb541d0 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java @@ -1,6 +1,5 @@ package eu.eudat.service.remotefetcher.config.entities; -import eu.eudat.service.remotefetcher.criteria.FetchStrategy; import eu.eudat.service.remotefetcher.config.UrlConfiguration; import jakarta.xml.bind.annotation.XmlElement; @@ -25,9 +24,4 @@ public class PrefillingSearch extends GenericUrls { urls.add(urlConfig); return urls; } - - @Override - public FetchStrategy getFetchMode() { - return null; - } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/criteria/FetchStrategy.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/criteria/FetchStrategy.java deleted file mode 100644 index a1204f77d..000000000 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/criteria/FetchStrategy.java +++ /dev/null @@ -1,8 +0,0 @@ -package eu.eudat.service.remotefetcher.criteria; - -public enum FetchStrategy { - - FIRST, - ALL - -} diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalValidationController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalValidationController.java index 2a0297821..1801bfcfc 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalValidationController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalValidationController.java @@ -1,7 +1,6 @@ package eu.eudat.controllers; import eu.eudat.authorization.Permission; -import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.logic.services.ExternalValidationService; import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.tools.exception.MyNotFoundException; @@ -23,7 +22,7 @@ public class ExternalValidationController { @GetMapping(path = {""}, produces = "application/json") public @ResponseBody Boolean validate(@RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type - ) throws HugeResultSetException, MyNotFoundException { + ) throws MyNotFoundException { this.authorizationService.authorizeForce(Permission.AuthenticatedRole); return this.externalValidationService.validateIdentifier(query, type); diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/ReferenceController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/ReferenceController.java index 684102bb4..7bd8d3a80 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/ReferenceController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/ReferenceController.java @@ -3,7 +3,6 @@ package eu.eudat.controllers; import com.fasterxml.jackson.core.JsonProcessingException; import eu.eudat.audit.AuditableAction; import eu.eudat.authorization.AuthorizationFlags; -import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.data.ReferenceEntity; import eu.eudat.model.Reference; import eu.eudat.model.builder.ReferenceBuilder; diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/old/Validation.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/old/Validation.java index 029940c2a..b702e9b9b 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/old/Validation.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/old/Validation.java @@ -2,7 +2,6 @@ package eu.eudat.controllers.old; import eu.eudat.authorization.Permission; import eu.eudat.logic.managers.ValidationManager; -import eu.eudat.commons.exceptions.HugeResultSetException; import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.tools.exception.MyNotFoundException; import org.springframework.beans.factory.annotation.Autowired; @@ -26,7 +25,7 @@ public class Validation { public @ResponseBody Boolean validate( @RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type - ) throws HugeResultSetException, MyNotFoundException { + ) throws MyNotFoundException { this.authorizationService.authorizeForce(Permission.AuthenticatedRole); Boolean isValid = this.validationManager.validateIdentifier(query, type); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ValidationManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ValidationManager.java index 8800ad717..27a37f8c9 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ValidationManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/ValidationManager.java @@ -6,8 +6,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; -import eu.eudat.commons.exceptions.HugeResultSetException; -//import eu.eudat.service.remotefetcher.RemoteFetcherService; @Component public class ValidationManager { @@ -22,7 +20,7 @@ public class ValidationManager { this.userScope = userScope; } - public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException, HugeResultSetException { + public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException { ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(identifier); // Integer count = this.remoteFetcherService.countEntries(externalReferenceCriteria, type); Integer count = 1; diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/services/ExternalValidationService.java b/dmp-backend/web/src/main/java/eu/eudat/logic/services/ExternalValidationService.java index eed16a8d7..d3906fd2a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/services/ExternalValidationService.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/services/ExternalValidationService.java @@ -2,8 +2,6 @@ package eu.eudat.logic.services; import eu.eudat.commons.scope.user.UserScope; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; -import eu.eudat.commons.exceptions.HugeResultSetException; -//import eu.eudat.service.remotefetcher.RemoteFetcherService; import gr.cite.tools.exception.MyNotFoundException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -21,7 +19,7 @@ public class ExternalValidationService { this.userScope = userScope; } - public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException, HugeResultSetException { + public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException { ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(identifier); // Integer count = this.remoteFetcherService.countEntries(externalReferenceCriteria, type); Integer count = 1;