diff --git a/dmp-backend/core/src/main/java/eu/eudat/data/ReferenceEntity.java b/dmp-backend/core/src/main/java/eu/eudat/data/ReferenceEntity.java index c3dcda35c..3c5cafb37 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/data/ReferenceEntity.java +++ b/dmp-backend/core/src/main/java/eu/eudat/data/ReferenceEntity.java @@ -14,6 +14,16 @@ import java.util.UUID; @Table(name = "\"Reference\"") public class ReferenceEntity extends TenantScopedBaseEntity { + public static class KnownFields { + public static final String Key = "key"; + public static final String ReferenceId = "reference_id"; + public static final String Abbreviation = "abbreviation"; + public static final String Description = "description"; + public static final String Label = "label"; + public static final String SourceLabel = "tag"; + } + + @Id @Column(name = "id", columnDefinition = "uuid", updatable = false, nullable = false) private UUID id; diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java index f78be7775..e3beeca31 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java @@ -201,14 +201,14 @@ public class ReferenceServiceImpl implements ReferenceService { @NotNull private ReferenceEntity buildReferenceEntityFromExternalData(Map result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) { - ReferenceEntity referenceEntity = new ReferenceEntity(); //TODO new reference logic + ReferenceEntity referenceEntity = new ReferenceEntity(); referenceEntity.setTypeId(data.getId()); referenceEntity.setIsActive(IsActive.Active); - referenceEntity.setReference(result.getOrDefault("key", null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("pid", null)); - referenceEntity.setSource(result.getOrDefault("tag", null)); - referenceEntity.setAbbreviation(result.getOrDefault("abbreviation", null)); - referenceEntity.setDescription(result.getOrDefault("description", null)); - referenceEntity.setLabel(result.getOrDefault("name", null)); + referenceEntity.setReference(result.getOrDefault(ReferenceEntity.KnownFields.Key, null) + ":" + remoteRepos.getResults().getFirst().getOrDefault("reference_id", null)); + referenceEntity.setSource(result.getOrDefault(ReferenceEntity.KnownFields.Key, null)); + referenceEntity.setAbbreviation(result.getOrDefault(ReferenceEntity.KnownFields.Abbreviation, null)); + referenceEntity.setDescription(result.getOrDefault(ReferenceEntity.KnownFields.Description, null)); + referenceEntity.setLabel(result.getOrDefault(ReferenceEntity.KnownFields.Abbreviation, null)); referenceEntity.setSourceType(ReferenceSourceType.External); DefinitionEntity definitionEntity = new DefinitionEntity(); definitionEntity.setFields(new ArrayList<>()); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java index ec47510c7..b7a09f536 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java @@ -5,10 +5,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; -import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.enums.ReferenceTypeSourceType; import eu.eudat.commons.exceptions.HugeResultSetException; import eu.eudat.convention.ConventionService; +import eu.eudat.data.ReferenceEntity; import eu.eudat.service.remotefetcher.config.entities.*; import eu.eudat.service.remotefetcher.models.ExternalDataResult; import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria; @@ -26,8 +26,6 @@ import org.springframework.stereotype.Service; import org.springframework.web.reactive.function.client.WebClient; import reactor.netty.http.client.HttpClient; -import java.io.File; -import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; @@ -38,12 +36,10 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { private WebClient webClient; private final ExternalUrlConfigProvider externalUrlConfigProvider; private final ConventionService conventionService; - private final XmlHandlingService xmlHandlingService; @Autowired - public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService, XmlHandlingService xmlHandlingService) { + public RemoteFetcherServiceImpl(ExternalUrlConfigProvider externalUrlConfigProvider, ConventionService conventionService) { this.externalUrlConfigProvider = externalUrlConfigProvider; this.conventionService = conventionService; - this.xmlHandlingService = xmlHandlingService; } private WebClient getWebClient(){ @@ -61,9 +57,9 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { @Override public ExternalDataResult getExternalData(List sources, ExternalReferenceCriteria externalReferenceCriteria, String key, FetchStrategy fetchStrategy) { - List apiSourcesToUse = sources.stream().map(x -> (SourceBaseConfiguration)x).toList(); + List apiSourcesToUse = sources; if (!this.conventionService.isNullOrEmpty(key)){ - apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).map(x -> (SourceBaseConfiguration)x).toList(); + apiSourcesToUse = sources.stream().filter(x-> x.getKey().equals(key)).toList(); } if (this.conventionService.isListNullOrEmpty(apiSourcesToUse)) return new ExternalDataResult(); @@ -99,11 +95,28 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { } } else if (source.getType() != null && source.getType().equals(ReferenceTypeSourceType.STATIC)) { SourceStaticOptionConfiguration staticSource = (SourceStaticOptionConfiguration)source; - results.addAll(getAllResultsFromMockUpJson(staticSource.getKey(), externalReferenceCriteria.getLike())); //TODO: + results.addAll(queryStaticData(staticSource, externalReferenceCriteria)); } } return results; } + private ExternalDataResult queryStaticData(SourceStaticOptionConfiguration staticSource, ExternalReferenceCriteria externalReferenceCriteria){ + Map result = new HashMap<>(); + Map rawData = new HashMap<>(); + for (StaticOption staticOption : staticSource.getOptions()){ + if (!this.conventionService.isNullOrEmpty(externalReferenceCriteria.getLike()) && !externalReferenceCriteria.getLike().toUpperCase().contains(staticOption.getValue())) continue; + result.put(staticOption.getCode(), staticOption.getValue()); + rawData.put(staticOption.getCode(), staticOption.getValue()); + result.put(ReferenceEntity.KnownFields.SourceLabel, staticSource.getLabel()); + result.put(ReferenceEntity.KnownFields.Key, staticSource.getKey()); + } + ExternalDataResult externalDataResult = new ExternalDataResult(); + externalDataResult.setRawData(new ArrayList<>()); + externalDataResult.setResults(new ArrayList<>()); + externalDataResult.getRawData().add(rawData); + externalDataResult.getResults().add(result); + return externalDataResult; + } private String buildAuthentication(AuthenticationConfiguration authenticationConfiguration) { HttpMethod method; @@ -210,7 +223,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { .filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase())) .collect(Collectors.toList())); } - externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put("tag", apiSource.getLabel())).peek(x -> x.put("key", apiSource.getKey())).toList()); + externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put(ReferenceEntity.KnownFields.SourceLabel, apiSource.getLabel())).peek(x -> x.put(ReferenceEntity.KnownFields.Key, apiSource.getKey())).toList()); if (fetchStrategy == FetchStrategy.FIRST) return externalDataResult; Long maxResults = this.externalUrlConfigProvider.getExternalUrls().getMaxresults(); @@ -252,7 +265,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { if (responseContentType.contains("json") ) { DocumentContext jsonContext = JsonPath.parse(response.getBody()); - return this.parseData(jsonContext, apiSource.getResults()); + return this.jsonToExternalDataResult(jsonContext, apiSource.getResults()); } else { throw new MyApplicationException("Unsupported response type" + responseContentType); } @@ -264,60 +277,35 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { return null; } - private ExternalDataResult parseData (DocumentContext jsonContext, ResultsConfiguration resultsConfigurationEntity) { + private ExternalDataResult jsonToExternalDataResult(DocumentContext jsonContext, ResultsConfiguration resultsConfigurationEntity) { ExternalDataResult result = new ExternalDataResult(); if (this.conventionService.isNullOrEmpty(resultsConfigurationEntity.getResultsArrayPath())) return new ExternalDataResult(); List> rawData = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath()); - result.setRawData(rawData); if (this.conventionService.isListNullOrEmpty(rawData) || this.conventionService.isListNullOrEmpty(resultsConfigurationEntity.getFieldsMapping())) return new ExternalDataResult(); + List results = jsonContext.read(resultsConfigurationEntity.getResultsArrayPath()); + List> parsedData = new ArrayList<>(); - for (Map stringObjectMap: rawData){ + for(Object resultItem : results){ Map map = new HashMap<>(); - for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()){ - String pathValue = field.getResponsePath(); - if (!pathValue.contains(".")){ - if (stringObjectMap.containsKey(pathValue)) { - map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue))); - } - }else { - if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){ - String value = null; - Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]); - if (fieldObj != null){ - if (fieldObj instanceof Map){ - Object o = ((Map) fieldObj).get(pathValue.split("\\.")[1]); - if(o instanceof String){ - value = (String)o; - } - else if(o instanceof Integer){ - value = String.valueOf(o); - } - } else if (fieldObj instanceof List) { - Object o = ((List>) fieldObj).get(0).get(pathValue.split("\\.")[1]); - if(o instanceof String){ - value = (String)o; - } - else if(o instanceof Integer){ - value = String.valueOf(o); - } - } - } - if (value != null){ - map.put(field.getCode(), value); - } - } + for(ResultFieldsMappingConfiguration field : resultsConfigurationEntity.getFieldsMapping()) { + StringBuilder pathValue = new StringBuilder(); + for (String pathPart : field.getResponsePath().split("\\.")) { + pathValue.append("['").append(pathPart).append("']"); } + Object value = JsonPath.parse(resultItem).read(pathValue.toString()); + map.put(field.getCode(), normalizeJsonValue(value)); } parsedData.add(map); } + result.setResults(parsedData); return result; } - private static String normalizeValue(Object value) { + private static String normalizeJsonValue(Object value) { if (value instanceof JSONArray jsonArray) { if (!jsonArray.isEmpty() && jsonArray.getFirst() instanceof String) { @@ -326,7 +314,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { for (Object o : jsonArray) { if ((o instanceof Map) && ((Map) o).containsKey("content")) { try { - return ((Map) o).get("content"); + return String.valueOf(((Map) o).get("content")); } catch (ClassCastException e){ if(((Map) o).get("content") instanceof Integer) { @@ -338,38 +326,11 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService { } } } else if (value instanceof Map) { - String key = ((Map)value).containsKey("$") ? "$" : "content"; - return ((Map)value).get(key); + String key = ((Map)value).containsKey("$") ? "$" : "content"; + return String.valueOf(((Map)value).get(key)); } return value != null ? value.toString() : null; } - - private ExternalDataResult getAllResultsFromMockUpJson(String path, String query) { - List> internalResults; - try { - String filePath = Paths.get(path).toUri().toURL().toString(); - ObjectMapper mapper = new ObjectMapper(); - internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); - return new ExternalDataResult(searchListMap(internalResults, query)); - } catch (Exception e) { - logger.error(e.getMessage(), e); - return new ExternalDataResult(); - } - } - - private List> searchListMap(List> internalResults, String query) { - List> list = new LinkedList<>(); - for (Map map : internalResults) - { - if (map.get("name") != null && map.get("name").toUpperCase().contains(query.toUpperCase())) { - list.add(map); - } - if (map.get("label") != null && map.get("label").toUpperCase().contains(query.toUpperCase())) { - list.add(map); - } - } - return list; - } private String parseBodyString(String bodyString) { String finalBodyString = bodyString; diff --git a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java index ba1dac251..a55bd84f4 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java +++ b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java @@ -12,6 +12,8 @@ import org.springframework.context.annotation.Primary; import org.springframework.http.converter.json.Jackson2ObjectMapperBuilder; import org.springframework.scheduling.annotation.EnableAsync; +import java.util.Map; + @SpringBootApplication(scanBasePackages = { "eu.eudat", "eu.eudat.depositinterface",