diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java index 598ce26a9..30068dee4 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/reference/ReferenceServiceImpl.java @@ -184,7 +184,7 @@ public class ReferenceServiceImpl implements ReferenceService { List referenceEntities = new ArrayList<>(); for (Map result : remoteRepos.getResults()){ if (result == null || result.isEmpty()) continue;; - ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data, remoteRepos); + ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data); referenceEntities.add(referenceEntity); } externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities); @@ -196,7 +196,7 @@ public class ReferenceServiceImpl implements ReferenceService { if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null && x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference); } - if (!this.conventionService.isNullOrEmpty(like)) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(like.toLowerCase())).collect(Collectors.toList()); } + if (!this.conventionService.isNullOrEmpty(like)) { models = models.stream().filter(x -> x.getLabel().toLowerCase(Locale.ROOT).toLowerCase().contains(like.toLowerCase(Locale.ROOT))).collect(Collectors.toList()); } models = models.stream().sorted(Comparator.comparing(Reference::getLabel, Comparator.nullsFirst(Comparator.naturalOrder()))).collect(Collectors.toList()); if (lookup.getPage() != null && !lookup.getPage().isEmpty()){ @@ -207,7 +207,7 @@ public class ReferenceServiceImpl implements ReferenceService { } @NotNull - private ReferenceEntity buildReferenceEntityFromExternalData(Map result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) { + private ReferenceEntity buildReferenceEntityFromExternalData(Map result, ReferenceTypeEntity data) { ReferenceEntity referenceEntity = new ReferenceEntity(); referenceEntity.setTypeId(data.getId()); referenceEntity.setIsActive(IsActive.Active); @@ -220,23 +220,26 @@ public class ReferenceServiceImpl implements ReferenceService { DefinitionEntity definitionEntity = new DefinitionEntity(); definitionEntity.setFields(new ArrayList<>()); for (Map.Entry resultValue : result.entrySet()){ + if (resultValue.getKey().equals(ReferenceEntity.KnownFields.ReferenceId) + || resultValue.getKey().equals(ReferenceEntity.KnownFields.Abbreviation) + || resultValue.getKey().equals(ReferenceEntity.KnownFields.Description) + || resultValue.getKey().equals(ReferenceEntity.KnownFields.Label) + || resultValue.getKey().equals(ReferenceEntity.KnownFields.SourceLabel)) continue; FieldEntity fieldEntity = new FieldEntity(); fieldEntity.setCode(resultValue.getKey()); fieldEntity.setValue(resultValue.getValue()); fieldEntity.setDataType(ReferenceFieldDataType.Text); definitionEntity.getFields().add(fieldEntity); } - referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity)); + if (!definitionEntity.getFields().isEmpty()) referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity)); return referenceEntity; } private List fetchReferenceFromDb(ReferenceSearchLookup lookup){ ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId()); - //TODO: Stratis update db for references. - //List data = query.collectAs(lookup.getProject()); - //return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data); - return new ArrayList<>(); + List data = query.collectAs(lookup.getProject()); + return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data); } private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) { @@ -250,257 +253,5 @@ public class ReferenceServiceImpl implements ReferenceService { return results; } -// private List> getAll ( List sources, ReferenceDefinitionSearchLookup lookup){ -// List> results = new LinkedList<>(); -// -// if (sources == null || sources.isEmpty()) { -// return results; -// } -// -// sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal)); -// -// List apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList(); -// apiSources.forEach(source -> { -// try { -// String auth = null; -// if (source.getAuth()!= null) { -// //auth = this.getAuthentication(source.getAuth()); -// } -// results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth)); -// } catch (Exception e) { -// logger.error(e.getLocalizedMessage(), e); -// } -// }); -// List staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList(); -// staticSources.forEach(source -> { -// Map map = new HashMap<>(); -// source.getOptions().forEach(option -> { -// map.put(option.getCode(), option.getValue()); -// map.put("tag", source.getLabel()); -// map.put("key", source.getKey()); -// }); -// results.add(map); -// }); -// -// return results; -// } - -// private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) { -// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name()); -// Map response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl()) -// .contentType(MediaType.APPLICATION_JSON) -// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody())) -// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference>() { -// })).block(); -// -// -// -// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath()); -// } - -// private String parseBodyString(String bodyString) { -// String finalBodyString = bodyString; -// if (bodyString.contains("{env:")) { -// int index = bodyString.indexOf("{env: "); -// while (index >= 0) { -// int endIndex = bodyString.indexOf("}", index + 6); -// String envName = bodyString.substring(index + 6, endIndex); -// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName)); -// index = bodyString.indexOf("{env: ", index + 6); -// } -// } -// return finalBodyString; -// } - -// private List> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List queries, String auth) throws Exception { -// Set pages = new HashSet<>(); -// -// String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries); -// String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries); -// -// ExternalDataResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth); -// if(externalRefernceResult != null) { -// if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) { -// externalRefernceResult.setResults(externalRefernceResult.getResults().stream() -// .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase())) -// .collect(Collectors.toList())); -// } -// if (fetchStrategy == FetchStrategy.FIRST) -// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList()); -// -// //Long maxResults = configLoader.getExternalUrls().getMaxresults(); -// Long maxResults = Long.valueOf(1000); -// if (externalRefernceResult.getResults().size() > maxResults) -// throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getResults().size() + " results... Please submit a more detailed search query"); -// -// Optional optionalResults = pages.parallelStream() -// .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth)) -// .filter(Objects::nonNull) -// .reduce((result1, result2) -> { -// result1.getResults().addAll(result2.getResults()); -// return result1; -// }); -// ExternalDataResult remainingExternalDataResult = optionalResults.orElseGet(ExternalDataResult::new); -// remainingExternalDataResult.getResults().addAll(externalRefernceResult.getResults()); -// -// return remainingExternalDataResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList()); -// } -// else { -// return new LinkedList<>(); -// } -// } - -// private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List queries){ -// String completedPath = urlPath; -// -// if (urlPath.contains("openaire") || urlPath.contains("orcid") ){ -// if (lookup.getLike() != null) { -// completedPath = completedPath.replace("{like}", lookup.getLike()); -// } else { -// completedPath = completedPath.replace("{like}", "*"); -// } -// } -// -// if (urlPath.contains("{like}")){ -// if (lookup.getLike() != null) { -// completedPath = completedPath.replace("{like}", lookup.getLike()); -// } else { -// completedPath = completedPath.replace("{like}", ""); -// } -// } -// -// if (urlPath.contains("{page}")) { -// if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) { -// completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset())); -// } else if (firstPage != null) { -// completedPath = completedPath.replace("{page}", firstPage); -// } else { -// completedPath = completedPath.replace("{page}", "1"); -// } -// } -// -// if (urlPath.contains("{pageSize}")){ -// if (lookup.getPage() != null && lookup.getPage().getSize() > 0) { -// completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize())); -// } else { -// completedPath = completedPath.replace("{pageSize}", "100"); -// } -// } -// -// -// return completedPath; -// } - -// protected ExternalDataResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) { -// -// try { -// ResponseEntity response; -// JsonNode jsonBody = new ObjectMapper().readTree(requestBody); -// -// response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> { -// if (contentType != null && !contentType.isEmpty()) { -// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); -// httpHeaders.setContentType(MediaType.valueOf(contentType)); -// } -// if (auth != null) { -// httpHeaders.set("Authorization", auth); -// } -// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block(); -// if (response.getStatusCode() == HttpStatus.OK) { // success -// ExternalDataResult externalDataResult = new ExternalDataResult(); -// if (response.getHeaders().get("Content-Type").get(0).contains("json")) { -// DocumentContext jsonContext = JsonPath.parse(response.getBody()); -// externalDataResult = this.getFromJson(jsonContext, resultsEntity); -// } -// -// return externalDataResult; -// } -// } catch (Exception exception) { -// logger.error(exception.getMessage(), exception); -// } -// -// return null; -// } - - -// public static ExternalDataResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { -// return new ExternalDataResult(parseData(jsonContext, resultsEntity)); -// } - - -// private static List> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { -// List > rawData = jsonContext.read(resultsEntity.getResultsArrayPath()); -// List> parsedData = new ArrayList<>(); -// -// for (Map stringObjectMap: rawData){ -// Map map = new HashMap<>(); -// for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){ -// String pathValue = field.getResponsePath(); -// if (!pathValue.contains(".")){ -// if (stringObjectMap.containsKey(pathValue)) { -// //map.put(field.getCode(), stringObjectMap.get(pathValue)); -// map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue))); -// } -// }else { -// if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){ -// String value = null; -// Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]); -// if (fieldObj != null){ -// if (fieldObj instanceof Map){ -// Object o = ((Map) fieldObj).get(pathValue.split("\\.")[1]); -// if(o instanceof String){ -// value = (String)o; -// } -// else if(o instanceof Integer){ -// value = String.valueOf(o); -// } -// } else if (fieldObj instanceof List) { -// Object o = ((List>) fieldObj).get(0).get(pathValue.split("\\.")[1]); -// if(o instanceof String){ -// value = (String)o; -// } -// else if(o instanceof Integer){ -// value = String.valueOf(o); -// } -// } -// } -// if (value != null){ -// map.put(field.getCode(), value); -// } -// } -// } -// } -// parsedData.add(map); -// } -// return parsedData; -// } -// -// -// private static String normalizeValue(Object value) { -// if (value instanceof JSONArray) { -// JSONArray jarr = (JSONArray) value; -// if (jarr.get(0) instanceof String) { -// return jarr.get(0).toString(); -// } else { -// for (Object o : jarr) { -// if ((o instanceof Map) && ((Map) o).containsKey("content")) { -// try { -// return ((Map) o).get("content"); -// } -// catch (ClassCastException e){ -// if(((Map) o).get("content") instanceof Integer) { -// return String.valueOf(((Map) o).get("content")); -// } -// return null; -// } -// } -// } -// } -// } else if (value instanceof Map) { -// String key = ((Map)value).containsKey("$") ? "$" : "content"; -// return ((Map)value).get(key); -// } -// return value != null ? value.toString() : null; -// } } diff --git a/dmp-backend/web/src/main/resources/config/application.yml b/dmp-backend/web/src/main/resources/config/application.yml index a60b25ebf..0a8821829 100644 --- a/dmp-backend/web/src/main/resources/config/application.yml +++ b/dmp-backend/web/src/main/resources/config/application.yml @@ -24,7 +24,6 @@ spring: optional:classpath:config/deposit.yml[.yml], optional:classpath:config/deposit-${spring.profiles.active}.yml[.yml], optional:file:../config/deposit-${spring.profiles.active}.yml[.yml], optional:classpath:config/errors.yml[.yml], optional:classpath:config/errors-${spring.profiles.active}.yml[.yml], optional:file:../config/errors-${spring.profiles.active}.yml[.yml], optional:classpath:config/storage.yml[.yml], optional:classpath:config/storage-${spring.profiles.active}.yml[.yml], optional:file:../config/storage-${spring.profiles.active}.yml[.yml], - optional:classpath:config/reference-type.yml[.yml], optional:classpath:config/reference-type-${spring.profiles.active}.yml[.yml], optional:file:../config/reference-type-${spring.profiles.active}.yml[.yml], optional:classpath:config/tenant.yml[.yml], optional:classpath:config/tenant-${spring.profiles.active}.yml[.yml], optional:file:../config/tenant-${spring.profiles.active}.yml[.yml], optional:classpath:config/queue.yml[.yml], optional:classpath:config/queue-${spring.profiles.active}.yml[.yml], optional:file:../config/queue-${spring.profiles.active}.yml[.yml], optional:classpath:config/notification.yml[.yml], optional:classpath:config/notification-${spring.profiles.active}.yml[.yml], optional:file:../config/notification-${spring.profiles.active}.yml[.yml],