update remote fetcher

dmp-backend/core/src/main/java/eu/eudat/commons/exceptions/HugeResultSetException.java

@@ -1,13 +0,0 @@
-package eu.eudat.commons.exceptions;
-
-public class HugeResultSetException extends Exception {
-
-    private static final long serialVersionUID = -6961447213733280563L;
-
-
-    public HugeResultSetException(String message) {
-        super(message);
-    }
-
-
-}

dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java

@@ -60,36 +60,27 @@ public class PrefillingServiceImpl implements PrefillingService {
     private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(PrefillingServiceImpl.class));
     private final EntityManager entityManager;
     private final BuilderFactory builderFactory;
-    private final QueryFactory queryFactory;
     private final ConventionService conventionService;
     private final MessageSource messageSource;
-    private final JsonHandlingService jsonHandlingService;
     private final ExternalUrlConfigProvider externalUrlConfigProvider;
     private final XmlHandlingService xmlHandlingService;
     private final ValidatorFactory validatorFactory;
-    private final RemoteFetcherService remoteFetcherService;
     @Autowired
     public PrefillingServiceImpl(
 		    EntityManager entityManager,
 		    BuilderFactory builderFactory,
-		    QueryFactory queryFactory,
 		    ConventionService conventionService,
 		    MessageSource messageSource,
-		    JsonHandlingService jsonHandlingService,
 		    ExternalUrlConfigProvider externalUrlConfigProvider,
 		    XmlHandlingService xmlHandlingService,
-		    ValidatorFactory validatorFactory, 
+		    ValidatorFactory validatorFactory) {
-            RemoteFetcherService remoteFetcherService) {
         this.entityManager = entityManager;
         this.builderFactory = builderFactory;
-        this.queryFactory = queryFactory;
         this.conventionService = conventionService;
         this.messageSource = messageSource;
-	    this.jsonHandlingService = jsonHandlingService;
 	    this.externalUrlConfigProvider = externalUrlConfigProvider;
 	    this.xmlHandlingService = xmlHandlingService;
 	    this.validatorFactory = validatorFactory;
-	    this.remoteFetcherService = remoteFetcherService;
     }
 
     @Override

dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java

@@ -1,13 +1,9 @@
 package eu.eudat.service.reference;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
-import eu.eudat.commons.exceptions.HugeResultSetException;
-import eu.eudat.data.ReferenceTypeEntity;
 import eu.eudat.model.Reference;
 import eu.eudat.model.persist.ReferencePersist;
 import eu.eudat.query.lookup.ReferenceSearchLookup;
-import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import eu.eudat.service.remotefetcher.models.ExternalDataResult;
 import gr.cite.tools.exception.MyApplicationException;
 import gr.cite.tools.exception.MyForbiddenException;
 import gr.cite.tools.exception.MyNotFoundException;

dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java

@@ -243,7 +243,7 @@ public class ReferenceServiceImpl implements ReferenceService {
         ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition());
         if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult();
 
-        ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).collect(Collectors.toList()), externalReferenceCriteria,  key, null);
+        ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).collect(Collectors.toList()), externalReferenceCriteria,  key);
         for (Map<String, String> result: results.getResults()) {
             result.put("referenceType", referenceType.getName());
         }

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java

@@ -1,14 +1,12 @@
 package eu.eudat.service.remotefetcher;
 
-import eu.eudat.data.ReferenceTypeEntity;
 import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
 import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
 import eu.eudat.service.remotefetcher.models.ExternalDataResult;
 
 import java.util.List;
 
 public interface RemoteFetcherService {
-	ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy);
+	ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
 	Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
 }

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java

@@ -7,7 +7,6 @@ import com.jayway.jsonpath.JsonPath;
 import com.jayway.jsonpath.PathNotFoundException;
 import eu.eudat.commons.JsonHandlingService;
 import eu.eudat.commons.enums.ReferenceTypeSourceType;
-import eu.eudat.commons.exceptions.HugeResultSetException;
 import eu.eudat.convention.ConventionService;
 import eu.eudat.data.ReferenceEntity;
 import eu.eudat.model.Reference;
@@ -15,7 +14,6 @@ import eu.eudat.model.referencedefinition.Field;
 import eu.eudat.service.remotefetcher.config.entities.*;
 import eu.eudat.service.remotefetcher.models.ExternalDataResult;
 import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
 import gr.cite.tools.exception.MyApplicationException;
 import net.minidev.json.JSONArray;
 import org.slf4j.Logger;
@@ -37,12 +35,10 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
     private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
 
     private WebClient webClient;
-    private final ExternalUrlConfigProvider externalUrlConfigProvider;
 	private final ConventionService conventionService;
     private final JsonHandlingService jsonHandlingService;
     @Autowired
-    public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, JsonHandlingService jsonHandlingService) {
+    public RemoteFetcherServiceImpl(ConventionService conventionService, JsonHandlingService jsonHandlingService) {
-	    this.externalUrlConfigProvider = externalUrlConfigProvider;
 	    this.conventionService = conventionService;
         this.jsonHandlingService = jsonHandlingService;
     }
@@ -61,7 +57,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
 
 
     @Override
-    public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy)  {
+    public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key)  {
         List<SourceBaseConfiguration> apiSourcesToUse =  sources;
         if (!this.conventionService.isNullOrEmpty(key)){
             apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).toList();
@@ -70,15 +66,15 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
 
         apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal));
 
-        return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria);
+        return this.queryExternalData(sources, externalReferenceCriteria);
     }
 
     @Override
     public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
-        return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size();
+        return getExternalData(sources, externalReferenceCriteria, key).getResults().size();
     }
 
-    private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
+    private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria) {
 
         ExternalDataResult results = new ExternalDataResult();
 
@@ -94,7 +90,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
 				    if (apiSource.getAuth() != null && apiSource.getAuth().getEnabled() != null && apiSource.getAuth().getEnabled()) {
 					    auth = this.buildAuthentication(apiSource.getAuth());
 				    }
-				    results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth));
+				    results.addAll(this.queryExternalData(apiSource, externalReferenceCriteria, auth));
 			    } catch (Exception e) {
 				    logger.error(e.getLocalizedMessage(), e);
 			    }
@@ -141,19 +137,6 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
         return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null);
     }
 
-
-//    private void applyFunderToQuery(SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) {
-//        //TODO new reference logic
-//	    if (apiSource.getFunderQuery() != null) {
-//		    if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) {
-//			    apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery()));
-//		    }
-//		    else {
-//			    apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", ""));
-//		    }
-//	    }
-//    }
-
     private String replaceLookupFieldQuery(String query, ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig<QueryCaseConfig>> queryConfigs) {
         String finalQuery = query;
         String likeValue = this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike()) ? "" : externalReferenceCriteria.getLike();
@@ -222,7 +205,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
         return completedPath;
     }
 
-    private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig<QueryCaseConfig>> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
+    private ExternalDataResult queryExternalData(final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig<QueryCaseConfig>> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
         String replacedPath = replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria);
         String replacedBody = replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria);
 
@@ -234,12 +217,6 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
                         .collect(Collectors.toList()));
             }
             externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put(ReferenceEntity.KnownFields.SourceLabel, apiSource.getLabel())).peek(x -> x.put(ReferenceEntity.KnownFields.Key, apiSource.getKey())).toList());
-            if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
-
-            Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
-            if (externalDataResult.getResults().size() > maxResults)
-                throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query");
-
             return externalDataResult;
         }
         else {
@@ -300,22 +277,24 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
         List<Map<String, String>> parsedData = new ArrayList<>();
         for(Object resultItem : results){
             Map<String, String> map = new HashMap<>();
+            boolean isValid = true;
             for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) {
                 if (this.conventionService.isNullOrEmpty(field.getResponsePath()) || this.conventionService.isNullOrEmpty(field.getCode())) continue;
                 try {
                     Object value = JsonPath.parse(resultItem).read(field.getResponsePath());
                     map.put(field.getCode(), normalizeJsonValue(value));
                 }catch (PathNotFoundException e){
-//                    List<Object> nextLevelResults = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath() + field.getResponsePath());
-//                    for(Object nextLevelResultValue : nextLevelResults) {
-//                        map.put(field.getCode(), normalizeJsonValue(nextLevelResultValue));
-//                    }
                     logger.debug("Json Path Error: " + e.getMessage() + " on source " + jsonHandlingService.toJsonSafe(resultItem));
-                    //TODO if value don't exist
+                    if (ReferenceEntity.KnownFields.ReferenceId.equals(field.getCode())) {
-                    continue;
+                        isValid = false;
+                        break;
+                    }
                 }
             }
-            parsedData.add(map);
+            if (this.conventionService.isNullOrEmpty(map.getOrDefault(ReferenceEntity.KnownFields.ReferenceId, null))){
+                logger.warn("Invalid reference on source " + jsonHandlingService.toJsonSafe(resultItem));
+            }
+            if (isValid) parsedData.add(map);
         }
         
         result.setResults(parsedData);

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java

@@ -15,19 +15,8 @@ public class ExternalUrls implements Serializable {
 
     private static final long serialVersionUID = -5076364662014107275L;
 
-    Long maxresults;
-
     Map<String, PrefillingConfig> prefillings;
 
-    public Long getMaxresults() {
-        return maxresults;
-    }
-
-    @XmlElement(name = "maxresults")
-    public void setMaxresults(Long maxresults) {
-        this.maxresults = maxresults;
-    }
-
     public Map<String, PrefillingConfig> getPrefillings() {
         return prefillings;
     }

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java

@@ -1,13 +1,11 @@
 package eu.eudat.service.remotefetcher.config.entities;
 
-import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
 
 import java.util.List;
 
 public abstract class GenericUrls {
 
     public abstract List<SourceBaseConfiguration> getUrls();
-    public abstract FetchStrategy getFetchMode();
 }
 
 

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java

@@ -1,6 +1,5 @@
 package eu.eudat.service.remotefetcher.config.entities;
 
-import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
 import eu.eudat.service.remotefetcher.config.UrlConfiguration;
 
 import jakarta.xml.bind.annotation.XmlElement;
@@ -25,9 +24,4 @@ public class PrefillingSearch extends GenericUrls {
         urls.add(urlConfig);
         return urls;
     }
-
-    @Override
-    public FetchStrategy getFetchMode() {
-        return null;
-    }
 }

dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/criteria/FetchStrategy.java

@@ -1,8 +0,0 @@
-package eu.eudat.service.remotefetcher.criteria;
-
-public enum FetchStrategy {
-
-    FIRST,
-    ALL
-
-}

dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalValidationController.java

@@ -1,7 +1,6 @@
 package eu.eudat.controllers;
 
 import eu.eudat.authorization.Permission;
-import eu.eudat.commons.exceptions.HugeResultSetException;
 import eu.eudat.logic.services.ExternalValidationService;
 import gr.cite.commons.web.authz.service.AuthorizationService;
 import gr.cite.tools.exception.MyNotFoundException;
@@ -23,7 +22,7 @@ public class ExternalValidationController {
 
     @GetMapping(path = {""}, produces = "application/json")
     public @ResponseBody Boolean validate(@RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type
-    ) throws HugeResultSetException, MyNotFoundException {
+    ) throws  MyNotFoundException {
         this.authorizationService.authorizeForce(Permission.AuthenticatedRole);
         
         return this.externalValidationService.validateIdentifier(query, type);

dmp-backend/web/src/main/java/eu/eudat/controllers/ReferenceController.java

@@ -3,7 +3,6 @@ package eu.eudat.controllers;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import eu.eudat.audit.AuditableAction;
 import eu.eudat.authorization.AuthorizationFlags;
-import eu.eudat.commons.exceptions.HugeResultSetException;
 import eu.eudat.data.ReferenceEntity;
 import eu.eudat.model.Reference;
 import eu.eudat.model.builder.ReferenceBuilder;

dmp-backend/web/src/main/java/eu/eudat/controllers/old/Validation.java

@@ -2,7 +2,6 @@ package eu.eudat.controllers.old;
 
 import eu.eudat.authorization.Permission;
 import eu.eudat.logic.managers.ValidationManager;
-import eu.eudat.commons.exceptions.HugeResultSetException;
 import gr.cite.commons.web.authz.service.AuthorizationService;
 import gr.cite.tools.exception.MyNotFoundException;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -26,7 +25,7 @@ public class Validation  {
 	public @ResponseBody
 	Boolean validate(
 			@RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type
-	) throws HugeResultSetException, MyNotFoundException {
+	) throws MyNotFoundException {
 		this.authorizationService.authorizeForce(Permission.AuthenticatedRole);
 
 		Boolean isValid = this.validationManager.validateIdentifier(query, type);

dmp-backend/web/src/main/java/eu/eudat/logic/managers/ValidationManager.java

@@ -6,8 +6,6 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
 import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import eu.eudat.commons.exceptions.HugeResultSetException;
-//import eu.eudat.service.remotefetcher.RemoteFetcherService;
 
 @Component
 public class ValidationManager {
@@ -22,7 +20,7 @@ public class ValidationManager {
 		this.userScope = userScope;
 	}
 	
-	public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException, HugeResultSetException {
+	public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException {
 		ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(identifier);
 //		Integer count = this.remoteFetcherService.countEntries(externalReferenceCriteria, type);
 		Integer count = 1;

dmp-backend/web/src/main/java/eu/eudat/logic/services/ExternalValidationService.java

@@ -2,8 +2,6 @@ package eu.eudat.logic.services;
 
 import eu.eudat.commons.scope.user.UserScope;
 import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
-import eu.eudat.commons.exceptions.HugeResultSetException;
-//import eu.eudat.service.remotefetcher.RemoteFetcherService;
 import gr.cite.tools.exception.MyNotFoundException;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -21,7 +19,7 @@ public class ExternalValidationService {
 		this.userScope = userScope;
 	}
 	
-	public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException, HugeResultSetException {
+	public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException {
 		ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(identifier);
 //		Integer count = this.remoteFetcherService.countEntries(externalReferenceCriteria, type);
 		Integer count = 1;