reference changes

This commit is contained in:
Efstratios Giannopoulos 2024-02-20 11:28:33 +02:00
parent d50613f416
commit 61560ec8ad
2 changed files with 11 additions and 261 deletions

View File

@ -184,7 +184,7 @@ public class ReferenceServiceImpl implements ReferenceService {
List<ReferenceEntity> referenceEntities = new ArrayList<>(); List<ReferenceEntity> referenceEntities = new ArrayList<>();
for (Map<String, String> result : remoteRepos.getResults()){ for (Map<String, String> result : remoteRepos.getResults()){
if (result == null || result.isEmpty()) continue;; if (result == null || result.isEmpty()) continue;;
ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data, remoteRepos); ReferenceEntity referenceEntity = buildReferenceEntityFromExternalData(result, data);
referenceEntities.add(referenceEntity); referenceEntities.add(referenceEntity);
} }
externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities); externalModels = this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), referenceEntities);
@ -196,7 +196,7 @@ public class ReferenceServiceImpl implements ReferenceService {
if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null && x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference); if (models.stream().noneMatch(x-> x.getReference() != null && x.getSource() != null && x.getReference().equals(externalReference.getReference()) && x.getSource().equals(externalReference.getSource()))) models.add(externalReference);
} }
if (!this.conventionService.isNullOrEmpty(like)) { models = models.stream().filter(x -> x.getLabel().toLowerCase().contains(like.toLowerCase())).collect(Collectors.toList()); } if (!this.conventionService.isNullOrEmpty(like)) { models = models.stream().filter(x -> x.getLabel().toLowerCase(Locale.ROOT).toLowerCase().contains(like.toLowerCase(Locale.ROOT))).collect(Collectors.toList()); }
models = models.stream().sorted(Comparator.comparing(Reference::getLabel, Comparator.nullsFirst(Comparator.naturalOrder()))).collect(Collectors.toList()); models = models.stream().sorted(Comparator.comparing(Reference::getLabel, Comparator.nullsFirst(Comparator.naturalOrder()))).collect(Collectors.toList());
if (lookup.getPage() != null && !lookup.getPage().isEmpty()){ if (lookup.getPage() != null && !lookup.getPage().isEmpty()){
@ -207,7 +207,7 @@ public class ReferenceServiceImpl implements ReferenceService {
} }
@NotNull @NotNull
private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data, ExternalDataResult remoteRepos) { private ReferenceEntity buildReferenceEntityFromExternalData(Map<String, String> result, ReferenceTypeEntity data) {
ReferenceEntity referenceEntity = new ReferenceEntity(); ReferenceEntity referenceEntity = new ReferenceEntity();
referenceEntity.setTypeId(data.getId()); referenceEntity.setTypeId(data.getId());
referenceEntity.setIsActive(IsActive.Active); referenceEntity.setIsActive(IsActive.Active);
@ -220,23 +220,26 @@ public class ReferenceServiceImpl implements ReferenceService {
DefinitionEntity definitionEntity = new DefinitionEntity(); DefinitionEntity definitionEntity = new DefinitionEntity();
definitionEntity.setFields(new ArrayList<>()); definitionEntity.setFields(new ArrayList<>());
for (Map.Entry<String, String> resultValue : result.entrySet()){ for (Map.Entry<String, String> resultValue : result.entrySet()){
if (resultValue.getKey().equals(ReferenceEntity.KnownFields.ReferenceId)
|| resultValue.getKey().equals(ReferenceEntity.KnownFields.Abbreviation)
|| resultValue.getKey().equals(ReferenceEntity.KnownFields.Description)
|| resultValue.getKey().equals(ReferenceEntity.KnownFields.Label)
|| resultValue.getKey().equals(ReferenceEntity.KnownFields.SourceLabel)) continue;
FieldEntity fieldEntity = new FieldEntity(); FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode(resultValue.getKey()); fieldEntity.setCode(resultValue.getKey());
fieldEntity.setValue(resultValue.getValue()); fieldEntity.setValue(resultValue.getValue());
fieldEntity.setDataType(ReferenceFieldDataType.Text); fieldEntity.setDataType(ReferenceFieldDataType.Text);
definitionEntity.getFields().add(fieldEntity); definitionEntity.getFields().add(fieldEntity);
} }
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity)); if (!definitionEntity.getFields().isEmpty()) referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
return referenceEntity; return referenceEntity;
} }
private List<Reference> fetchReferenceFromDb(ReferenceSearchLookup lookup){ private List<Reference> fetchReferenceFromDb(ReferenceSearchLookup lookup){
ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId()); ReferenceQuery query = lookup.enrich(this.queryFactory).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).sourceTypes(ReferenceSourceType.Internal).typeIds(lookup.getTypeId());
//TODO: Stratis update db for references.
//List<ReferenceEntity> data = query.collectAs(lookup.getProject()); List<ReferenceEntity> data = query.collectAs(lookup.getProject());
//return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data); return this.builderFactory.builder(ReferenceBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(lookup.getProject(), data);
return new ArrayList<>();
} }
private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) { private ExternalDataResult getReferenceData(ReferenceTypeEntity referenceType, ExternalReferenceCriteria externalReferenceCriteria, String key) {
@ -250,257 +253,5 @@ public class ReferenceServiceImpl implements ReferenceService {
return results; return results;
} }
// private List<Map<String, String>> getAll ( List<ReferenceTypeSourceBaseConfigurationEntity> sources, ReferenceDefinitionSearchLookup lookup){
// List<Map<String, String>> results = new LinkedList<>();
//
// if (sources == null || sources.isEmpty()) {
// return results;
// }
//
// sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal));
//
// List<ReferenceTypeSourceExternalApiConfigurationEntity> apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList();
// apiSources.forEach(source -> {
// try {
// String auth = null;
// if (source.getAuth()!= null) {
// //auth = this.getAuthentication(source.getAuth());
// }
// results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth));
// } catch (Exception e) {
// logger.error(e.getLocalizedMessage(), e);
// }
// });
// List<ReferenceTypeSourceStaticOptionConfigurationEntity> staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList();
// staticSources.forEach(source -> {
// Map<String, String> map = new HashMap<>();
// source.getOptions().forEach(option -> {
// map.put(option.getCode(), option.getValue());
// map.put("tag", source.getLabel());
// map.put("key", source.getKey());
// });
// results.add(map);
// });
//
// return results;
// }
// private String getAuthentication(AuthenticationConfigurationEntity authenticationConfiguration) {
// HttpMethod method = HttpMethod.valueOf(authenticationConfiguration.getAuthMethod().name());
// Map<String, Object> response = this.client.method(method).uri(authenticationConfiguration.getAuthUrl())
// .contentType(MediaType.APPLICATION_JSON)
// .bodyValue(this.parseBodyString(authenticationConfiguration.getAuthRequestBody()))
// .exchangeToMono(mono -> mono.bodyToMono(new ParameterizedTypeReference<Map<String, Object>>() {
// })).block();
//
//
//
// return authenticationConfiguration.getType() + " " + response.get(authenticationConfiguration.getAuthTokenPath());
// }
// private String parseBodyString(String bodyString) {
// String finalBodyString = bodyString;
// if (bodyString.contains("{env:")) {
// int index = bodyString.indexOf("{env: ");
// while (index >= 0) {
// int endIndex = bodyString.indexOf("}", index + 6);
// String envName = bodyString.substring(index + 6, endIndex);
// finalBodyString = finalBodyString.replace("{env: " + envName + "}", System.getenv(envName));
// index = bodyString.indexOf("{env: ", index + 6);
// }
// }
// return finalBodyString;
// }
// private List<Map<String, String>> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List<QueryConfigEntity> queries, String auth) throws Exception {
// Set<Integer> pages = new HashSet<>();
//
// String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries);
// String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries);
//
// ExternalDataResult externalRefernceResult = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth);
// if(externalRefernceResult != null) {
// if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) {
// externalRefernceResult.setResults(externalRefernceResult.getResults().stream()
// .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase()))
// .collect(Collectors.toList()));
// }
// if (fetchStrategy == FetchStrategy.FIRST)
// return externalRefernceResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
//
// //Long maxResults = configLoader.getExternalUrls().getMaxresults();
// Long maxResults = Long.valueOf(1000);
// if (externalRefernceResult.getResults().size() > maxResults)
// throw new HugeResultSetException("The submitted search query " + lookup.getLike() + " is about to return " + externalRefernceResult.getResults().size() + " results... Please submit a more detailed search query");
//
// Optional<ExternalDataResult> optionalResults = pages.parallelStream()
// .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth))
// .filter(Objects::nonNull)
// .reduce((result1, result2) -> {
// result1.getResults().addAll(result2.getResults());
// return result1;
// });
// ExternalDataResult remainingExternalDataResult = optionalResults.orElseGet(ExternalDataResult::new);
// remainingExternalDataResult.getResults().addAll(externalRefernceResult.getResults());
//
// return remainingExternalDataResult.getResults().stream().peek(x -> x.put("tag", label)).peek(x -> x.put("key", key)).collect(Collectors.toList());
// }
// else {
// return new LinkedList<>();
// }
// }
// private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List<QueryConfigEntity> queries){
// String completedPath = urlPath;
//
// if (urlPath.contains("openaire") || urlPath.contains("orcid") ){
// if (lookup.getLike() != null) {
// completedPath = completedPath.replace("{like}", lookup.getLike());
// } else {
// completedPath = completedPath.replace("{like}", "*");
// }
// }
//
// if (urlPath.contains("{like}")){
// if (lookup.getLike() != null) {
// completedPath = completedPath.replace("{like}", lookup.getLike());
// } else {
// completedPath = completedPath.replace("{like}", "");
// }
// }
//
// if (urlPath.contains("{page}")) {
// if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) {
// completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset()));
// } else if (firstPage != null) {
// completedPath = completedPath.replace("{page}", firstPage);
// } else {
// completedPath = completedPath.replace("{page}", "1");
// }
// }
//
// if (urlPath.contains("{pageSize}")){
// if (lookup.getPage() != null && lookup.getPage().getSize() > 0) {
// completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize()));
// } else {
// completedPath = completedPath.replace("{pageSize}", "100");
// }
// }
//
//
// return completedPath;
// }
// protected ExternalDataResult getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) {
//
// try {
// ResponseEntity<String> response;
// JsonNode jsonBody = new ObjectMapper().readTree(requestBody);
//
// response = this.client.method(HttpMethod.valueOf(httpMethod.name())).uri(urlString).headers(httpHeaders -> {
// if (contentType != null && !contentType.isEmpty()) {
// httpHeaders.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
// httpHeaders.setContentType(MediaType.valueOf(contentType));
// }
// if (auth != null) {
// httpHeaders.set("Authorization", auth);
// }
// }).bodyValue(jsonBody).retrieve().toEntity(String.class).block();
// if (response.getStatusCode() == HttpStatus.OK) { // success
// ExternalDataResult externalDataResult = new ExternalDataResult();
// if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
// DocumentContext jsonContext = JsonPath.parse(response.getBody());
// externalDataResult = this.getFromJson(jsonContext, resultsEntity);
// }
//
// return externalDataResult;
// }
// } catch (Exception exception) {
// logger.error(exception.getMessage(), exception);
// }
//
// return null;
// }
// public static ExternalDataResult getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
// return new ExternalDataResult(parseData(jsonContext, resultsEntity));
// }
// private static List<Map<String, String>> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) {
// List <Map<String, String>> rawData = jsonContext.read(resultsEntity.getResultsArrayPath());
// List<Map<String, String>> parsedData = new ArrayList<>();
//
// for (Map<String, String> stringObjectMap: rawData){
// Map<String, String> map = new HashMap<>();
// for(ResultFieldsMappingConfigurationEntity field: resultsEntity.getFieldsMapping()){
// String pathValue = field.getResponsePath();
// if (!pathValue.contains(".")){
// if (stringObjectMap.containsKey(pathValue)) {
// //map.put(field.getCode(), stringObjectMap.get(pathValue));
// map.put(field.getCode(), normalizeValue(stringObjectMap.get(pathValue)));
// }
// }else {
// if (stringObjectMap.containsKey(pathValue.split("\\.")[0])){
// String value = null;
// Object fieldObj = stringObjectMap.get(pathValue.split("\\.")[0]);
// if (fieldObj != null){
// if (fieldObj instanceof Map){
// Object o = ((Map<String, Object>) fieldObj).get(pathValue.split("\\.")[1]);
// if(o instanceof String){
// value = (String)o;
// }
// else if(o instanceof Integer){
// value = String.valueOf(o);
// }
// } else if (fieldObj instanceof List) {
// Object o = ((List<Map<String,?>>) fieldObj).get(0).get(pathValue.split("\\.")[1]);
// if(o instanceof String){
// value = (String)o;
// }
// else if(o instanceof Integer){
// value = String.valueOf(o);
// }
// }
// }
// if (value != null){
// map.put(field.getCode(), value);
// }
// }
// }
// }
// parsedData.add(map);
// }
// return parsedData;
// }
//
//
// private static String normalizeValue(Object value) {
// if (value instanceof JSONArray) {
// JSONArray jarr = (JSONArray) value;
// if (jarr.get(0) instanceof String) {
// return jarr.get(0).toString();
// } else {
// for (Object o : jarr) {
// if ((o instanceof Map) && ((Map) o).containsKey("content")) {
// try {
// return ((Map<String, String>) o).get("content");
// }
// catch (ClassCastException e){
// if(((Map<?, ?>) o).get("content") instanceof Integer) {
// return String.valueOf(((Map<?, ?>) o).get("content"));
// }
// return null;
// }
// }
// }
// }
// } else if (value instanceof Map) {
// String key = ((Map<String, String>)value).containsKey("$") ? "$" : "content";
// return ((Map<String, String>)value).get(key);
// }
// return value != null ? value.toString() : null;
// }
} }

View File

@ -24,7 +24,6 @@ spring:
optional:classpath:config/deposit.yml[.yml], optional:classpath:config/deposit-${spring.profiles.active}.yml[.yml], optional:file:../config/deposit-${spring.profiles.active}.yml[.yml], optional:classpath:config/deposit.yml[.yml], optional:classpath:config/deposit-${spring.profiles.active}.yml[.yml], optional:file:../config/deposit-${spring.profiles.active}.yml[.yml],
optional:classpath:config/errors.yml[.yml], optional:classpath:config/errors-${spring.profiles.active}.yml[.yml], optional:file:../config/errors-${spring.profiles.active}.yml[.yml], optional:classpath:config/errors.yml[.yml], optional:classpath:config/errors-${spring.profiles.active}.yml[.yml], optional:file:../config/errors-${spring.profiles.active}.yml[.yml],
optional:classpath:config/storage.yml[.yml], optional:classpath:config/storage-${spring.profiles.active}.yml[.yml], optional:file:../config/storage-${spring.profiles.active}.yml[.yml], optional:classpath:config/storage.yml[.yml], optional:classpath:config/storage-${spring.profiles.active}.yml[.yml], optional:file:../config/storage-${spring.profiles.active}.yml[.yml],
optional:classpath:config/reference-type.yml[.yml], optional:classpath:config/reference-type-${spring.profiles.active}.yml[.yml], optional:file:../config/reference-type-${spring.profiles.active}.yml[.yml],
optional:classpath:config/tenant.yml[.yml], optional:classpath:config/tenant-${spring.profiles.active}.yml[.yml], optional:file:../config/tenant-${spring.profiles.active}.yml[.yml], optional:classpath:config/tenant.yml[.yml], optional:classpath:config/tenant-${spring.profiles.active}.yml[.yml], optional:file:../config/tenant-${spring.profiles.active}.yml[.yml],
optional:classpath:config/queue.yml[.yml], optional:classpath:config/queue-${spring.profiles.active}.yml[.yml], optional:file:../config/queue-${spring.profiles.active}.yml[.yml], optional:classpath:config/queue.yml[.yml], optional:classpath:config/queue-${spring.profiles.active}.yml[.yml], optional:file:../config/queue-${spring.profiles.active}.yml[.yml],
optional:classpath:config/notification.yml[.yml], optional:classpath:config/notification-${spring.profiles.active}.yml[.yml], optional:file:../config/notification-${spring.profiles.active}.yml[.yml], optional:classpath:config/notification.yml[.yml], optional:classpath:config/notification-${spring.profiles.active}.yml[.yml], optional:file:../config/notification-${spring.profiles.active}.yml[.yml],