MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

update remote fetcher

This commit is contained in:
Efstratios Giannopoulos 2024-02-19 18:10:34 +02:00
parent a44fe43cf3
commit 9f36710276
15 changed files with 23 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
dmp-backend/core/src/main/java/eu/eudat/commons/exceptions/HugeResultSetException.java
View File

@ -1,13 +0,0 @@
package eu.eudat.commons.exceptions;
public class HugeResultSetException extends Exception {
private static final long serialVersionUID = -6961447213733280563L;
public HugeResultSetException(String message) {
super(message);
}
}

11
dmp-backend/core/src/main/java/eu/eudat/service/prefilling/PrefillingServiceImpl.java
View File

@ -60,36 +60,27 @@ public class PrefillingServiceImpl implements PrefillingService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(PrefillingServiceImpl.class));
private final EntityManager entityManager;
private final BuilderFactory builderFactory;
private final QueryFactory queryFactory;
private final ConventionService conventionService;
private final MessageSource messageSource;
private final JsonHandlingService jsonHandlingService;
private final ExternalUrlConfigProvider externalUrlConfigProvider;
private final XmlHandlingService xmlHandlingService;
private final ValidatorFactory validatorFactory;
private final RemoteFetcherService remoteFetcherService;
@Autowired
public PrefillingServiceImpl(
EntityManager entityManager,
BuilderFactory builderFactory,
QueryFactory queryFactory,
ConventionService conventionService,
MessageSource messageSource,
JsonHandlingService jsonHandlingService,
ExternalUrlConfigProvider externalUrlConfigProvider,
XmlHandlingService xmlHandlingService,
ValidatorFactory validatorFactory,
RemoteFetcherService remoteFetcherService) {
ValidatorFactory validatorFactory) {
this.entityManager = entityManager;
this.builderFactory = builderFactory;
this.queryFactory = queryFactory;
this.conventionService = conventionService;
this.messageSource = messageSource;
this.jsonHandlingService = jsonHandlingService;
this.externalUrlConfigProvider = externalUrlConfigProvider;
this.xmlHandlingService = xmlHandlingService;
this.validatorFactory = validatorFactory;
this.remoteFetcherService = remoteFetcherService;
}
@Override

4
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceService.java
View File

@ -1,13 +1,9 @@
package eu.eudat.service.reference;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.Reference;
import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.query.lookup.ReferenceSearchLookup;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException;

2
dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java
View File

@ -243,7 +243,7 @@ public class ReferenceServiceImpl implements ReferenceService {
ReferenceTypeDefinitionEntity referenceTypeDefinition = this.xmlHandlingService.fromXmlSafe(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition());
if (referenceTypeDefinition == null || this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) return new ExternalDataResult();
ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).collect(Collectors.toList()), externalReferenceCriteria, key, null);
ExternalDataResult results = this.remoteFetcherService.getExternalData(referenceTypeDefinition.getSources().stream().map(x -> (SourceBaseConfiguration)x).collect(Collectors.toList()), externalReferenceCriteria, key);
for (Map<String, String> result: results.getResults()) {
result.put("referenceType", referenceType.getName());
}

4
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherService.java
View File

@ -1,14 +1,12 @@
package eu.eudat.service.remotefetcher;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.service.remotefetcher.config.entities.SourceBaseConfiguration;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import java.util.List;
public interface RemoteFetcherService {
ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy);
ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key);
}

53
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java
View File

@ -7,7 +7,6 @@ import com.jayway.jsonpath.JsonPath;
import com.jayway.jsonpath.PathNotFoundException;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.enums.ReferenceTypeSourceType;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.model.Reference;
@ -15,7 +14,6 @@ import eu.eudat.model.referencedefinition.Field;
import eu.eudat.service.remotefetcher.config.entities.*;
import eu.eudat.service.remotefetcher.models.ExternalDataResult;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import gr.cite.tools.exception.MyApplicationException;
import net.minidev.json.JSONArray;
import org.slf4j.Logger;
@ -37,12 +35,10 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
private static final Logger logger = LoggerFactory.getLogger(RemoteFetcherServiceImpl.class);
private WebClient webClient;
private final ExternalUrlConfigProvider externalUrlConfigProvider;
private final ConventionService conventionService;
private final JsonHandlingService jsonHandlingService;
@Autowired
public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, JsonHandlingService jsonHandlingService) {
this.externalUrlConfigProvider = externalUrlConfigProvider;
public RemoteFetcherServiceImpl(ConventionService conventionService, JsonHandlingService jsonHandlingService) {
this.conventionService = conventionService;
this.jsonHandlingService = jsonHandlingService;
}
@ -61,7 +57,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
@Override
public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) {
public ExternalDataResult getExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
List<SourceBaseConfiguration> apiSourcesToUse = sources;
if (!this.conventionService.isNullOrEmpty(key)){
apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).toList();
@ -70,15 +66,15 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
apiSourcesToUse.sort(Comparator.comparing(SourceBaseConfiguration::getOrdinal));
return this.queryExternalData(sources, fetchStrategy, externalReferenceCriteria);
return this.queryExternalData(sources, externalReferenceCriteria);
}
@Override
public Integer countExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria, String key) {
return getExternalData(sources, externalReferenceCriteria, key, null).getResults().size();
return getExternalData(sources, externalReferenceCriteria, key).getResults().size();
}
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, FetchStrategy fetchStrategy, ExternalReferenceCriteria externalReferenceCriteria) {
private ExternalDataResult queryExternalData(List<SourceBaseConfiguration> sources, ExternalReferenceCriteria externalReferenceCriteria) {
ExternalDataResult results = new ExternalDataResult();
@ -94,7 +90,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
if (apiSource.getAuth() != null && apiSource.getAuth().getEnabled() != null && apiSource.getAuth().getEnabled()) {
auth = this.buildAuthentication(apiSource.getAuth());
}
results.addAll(this.queryExternalData(fetchStrategy, apiSource, externalReferenceCriteria, auth));
results.addAll(this.queryExternalData(apiSource, externalReferenceCriteria, auth));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
@ -141,19 +137,6 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
return authenticationConfiguration.getType() + " " + response.getOrDefault(authenticationConfiguration.getAuthTokenPath(), null);
}
// private void applyFunderToQuery(SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig> apiSource, ExternalReferenceCriteria externalReferenceCriteria) {
// //TODO new reference logic
// if (apiSource.getFunderQuery() != null) {
// if (externalReferenceCriteria.getFunderId() != null && !apiSource.getFunderQuery().startsWith("dmp:")) {
// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", apiSource.getFunderQuery()));
// }
// else {
// apiSource.setUrl(apiSource.getUrl().replace("{funderQuery}", ""));
// }
// }
// }
private String replaceLookupFieldQuery(String query, ExternalReferenceCriteria externalReferenceCriteria, List<QueryConfig<QueryCaseConfig>> queryConfigs) {
String finalQuery = query;
String likeValue = this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike()) ? "" : externalReferenceCriteria.getLike();
@ -222,7 +205,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
return completedPath;
}
private ExternalDataResult queryExternalData(FetchStrategy fetchStrategy, final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig<QueryCaseConfig>> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
private ExternalDataResult queryExternalData(final SourceExternalApiConfiguration<ResultsConfiguration<ResultFieldsMappingConfiguration>, AuthenticationConfiguration, QueryConfig<QueryCaseConfig>> apiSource, ExternalReferenceCriteria externalReferenceCriteria, String auth) throws Exception {
String replacedPath = replaceLookupFields(apiSource.getUrl(), apiSource, externalReferenceCriteria);
String replacedBody = replaceLookupFields(apiSource.getRequestBody(), apiSource, externalReferenceCriteria);
@ -234,12 +217,6 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
.collect(Collectors.toList()));
}
externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put(ReferenceEntity.KnownFields.SourceLabel, apiSource.getLabel())).peek(x -> x.put(ReferenceEntity.KnownFields.Key, apiSource.getKey())).toList());
if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult;
Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults();
if (externalDataResult.getResults().size() > maxResults)
throw new HugeResultSetException("The submitted search query " + externalReferenceCriteria.getLike() + " is about to return " + externalDataResult.getResults().size() + " results... Please submit a more detailed search query");
return externalDataResult;
}
else {
@ -300,22 +277,24 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
List<Map<String, String>> parsedData = new ArrayList<>();
for(Object resultItem : results){
Map<String, String> map = new HashMap<>();
boolean isValid = true;
for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) {
if (this.conventionService.isNullOrEmpty(field.getResponsePath()) || this.conventionService.isNullOrEmpty(field.getCode())) continue;
try {
Object value = JsonPath.parse(resultItem).read(field.getResponsePath());
map.put(field.getCode(), normalizeJsonValue(value));
}catch (PathNotFoundException e){
// List<Object> nextLevelResults = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath() + field.getResponsePath());
// for(Object nextLevelResultValue : nextLevelResults) {
// map.put(field.getCode(), normalizeJsonValue(nextLevelResultValue));
// }
logger.debug("Json Path Error: " + e.getMessage() + " on source " + jsonHandlingService.toJsonSafe(resultItem));
//TODO if value don't exist
continue;
if (ReferenceEntity.KnownFields.ReferenceId.equals(field.getCode())) {
isValid = false;
break;
}
}
}
parsedData.add(map);
if (this.conventionService.isNullOrEmpty(map.getOrDefault(ReferenceEntity.KnownFields.ReferenceId, null))){
logger.warn("Invalid reference on source " + jsonHandlingService.toJsonSafe(resultItem));
}
if (isValid) parsedData.add(map);
}
result.setResults(parsedData);

11
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/ExternalUrls.java
View File

@ -15,19 +15,8 @@ public class ExternalUrls implements Serializable {
private static final long serialVersionUID = -5076364662014107275L;
Long maxresults;
Map<String, PrefillingConfig> prefillings;
public Long getMaxresults() {
return maxresults;
}
@XmlElement(name = "maxresults")
public void setMaxresults(Long maxresults) {
this.maxresults = maxresults;
}
public Map<String, PrefillingConfig> getPrefillings() {
return prefillings;
}

2
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/GenericUrls.java
View File

@ -1,13 +1,11 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import java.util.List;
public abstract class GenericUrls {
public abstract List<SourceBaseConfiguration> getUrls();
public abstract FetchStrategy getFetchMode();
}

6
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/config/entities/PrefillingSearch.java
View File

@ -1,6 +1,5 @@
package eu.eudat.service.remotefetcher.config.entities;
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
import jakarta.xml.bind.annotation.XmlElement;
@ -25,9 +24,4 @@ public class PrefillingSearch extends GenericUrls {
urls.add(urlConfig);
return urls;
}
@Override
public FetchStrategy getFetchMode() {
return null;
}
}

8
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/criteria/FetchStrategy.java
View File

@ -1,8 +0,0 @@
package eu.eudat.service.remotefetcher.criteria;
public enum FetchStrategy {
FIRST,
ALL
}

3
dmp-backend/web/src/main/java/eu/eudat/controllers/ExternalValidationController.java
View File

@ -1,7 +1,6 @@
package eu.eudat.controllers;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.logic.services.ExternalValidationService;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.exception.MyNotFoundException;
@ -23,7 +22,7 @@ public class ExternalValidationController {
@GetMapping(path = {""}, produces = "application/json")
public @ResponseBody Boolean validate(@RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type
) throws HugeResultSetException, MyNotFoundException {
) throws MyNotFoundException {
this.authorizationService.authorizeForce(Permission.AuthenticatedRole);
return this.externalValidationService.validateIdentifier(query, type);

1
dmp-backend/web/src/main/java/eu/eudat/controllers/ReferenceController.java
View File

@ -3,7 +3,6 @@ package eu.eudat.controllers;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.audit.AuditableAction;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.commons.exceptions.HugeResultSetException;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.model.Reference;
import eu.eudat.model.builder.ReferenceBuilder;

3
dmp-backend/web/src/main/java/eu/eudat/controllers/old/Validation.java
View File

@ -2,7 +2,6 @@ package eu.eudat.controllers.old;
import eu.eudat.authorization.Permission;
import eu.eudat.logic.managers.ValidationManager;
import eu.eudat.commons.exceptions.HugeResultSetException;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.exception.MyNotFoundException;
import org.springframework.beans.factory.annotation.Autowired;
@ -26,7 +25,7 @@ public class Validation {
public @ResponseBody
Boolean validate(
@RequestParam(value = "query", required = false) String query, @RequestParam(value = "type", required = false) String type
) throws HugeResultSetException, MyNotFoundException {
) throws MyNotFoundException {
this.authorizationService.authorizeForce(Permission.AuthenticatedRole);
Boolean isValid = this.validationManager.validateIdentifier(query, type);

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ValidationManager.java
View File

@ -6,8 +6,6 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.commons.exceptions.HugeResultSetException;
//import eu.eudat.service.remotefetcher.RemoteFetcherService;
@Component
public class ValidationManager {
@ -22,7 +20,7 @@ public class ValidationManager {
this.userScope = userScope;
}
public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException, HugeResultSetException {
public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException {
ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(identifier);
// Integer count = this.remoteFetcherService.countEntries(externalReferenceCriteria, type);
Integer count = 1;

4
dmp-backend/web/src/main/java/eu/eudat/logic/services/ExternalValidationService.java
View File

@ -2,8 +2,6 @@ package eu.eudat.logic.services;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
import eu.eudat.commons.exceptions.HugeResultSetException;
//import eu.eudat.service.remotefetcher.RemoteFetcherService;
import gr.cite.tools.exception.MyNotFoundException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@ -21,7 +19,7 @@ public class ExternalValidationService {
this.userScope = userScope;
}
public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException, HugeResultSetException {
public Boolean validateIdentifier(String identifier, String type) throws MyNotFoundException {
ExternalReferenceCriteria externalReferenceCriteria = new ExternalReferenceCriteria(identifier);
// Integer count = this.remoteFetcherService.countEntries(externalReferenceCriteria, type);
Integer count = 1;