From f2ee851baba3d5dee816dabac3d254043fb04373 Mon Sep 17 00:00:00 2001 From: amentis Date: Thu, 16 Nov 2023 15:40:25 +0200 Subject: [PATCH] reference type persist and editor --- .../java/eu/eudat/model/ReferenceType.java | 11 +++ .../model/builder/ReferenceTypeBuilder.java | 4 +- .../ReferenceTypeServiceImpl.java | 6 +- .../services/references/ReferenceService.java | 94 ++++++++++++++----- .../reference-type-editor.component.html | 4 +- .../editor/reference-type-editor.component.ts | 65 ++++++++++--- .../editor/reference-type-editor.model.ts | 4 +- .../dmp-editor-blueprint.component.ts | 4 +- 8 files changed, 146 insertions(+), 46 deletions(-) diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/ReferenceType.java b/dmp-backend/core/src/main/java/eu/eudat/model/ReferenceType.java index 76aa193f8..02e8e7941 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/model/ReferenceType.java +++ b/dmp-backend/core/src/main/java/eu/eudat/model/ReferenceType.java @@ -29,6 +29,9 @@ public class ReferenceType { private Instant updatedAt; public static final String _updatedAt = "updatedAt"; + public final static String _hash = "hash"; + private String hash; + public UUID getId() { return id; } @@ -84,4 +87,12 @@ public class ReferenceType { public void setUpdatedAt(Instant updatedAt) { this.updatedAt = updatedAt; } + + public String getHash() { + return hash; + } + + public void setHash(String hash) { + this.hash = hash; + } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/model/builder/ReferenceTypeBuilder.java b/dmp-backend/core/src/main/java/eu/eudat/model/builder/ReferenceTypeBuilder.java index 9379bad25..440b88322 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/model/builder/ReferenceTypeBuilder.java +++ b/dmp-backend/core/src/main/java/eu/eudat/model/builder/ReferenceTypeBuilder.java @@ -2,14 +2,11 @@ package eu.eudat.model.builder; import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.commons.XmlHandlingService; -import eu.eudat.commons.types.reference.DefinitionEntity; import eu.eudat.commons.types.referencetype.ReferenceTypeDefinitionEntity; import eu.eudat.convention.ConventionService; import eu.eudat.data.ReferenceTypeEntity; import eu.eudat.model.ReferenceType; -import eu.eudat.model.builder.referencedefinition.DefinitionBuilder; import eu.eudat.model.builder.referencetypedefinition.ReferenceTypeDefinitionBuilder; -import eu.eudat.model.referencetypedefinition.ReferenceTypeDefinition; import gr.cite.tools.data.builder.BuilderFactory; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.exception.MyApplicationException; @@ -66,6 +63,7 @@ public class ReferenceTypeBuilder extends BaseBuilder> remoteRepos = this.getAll(definition.getSources()); + List> remoteRepos = this.getAll(definition.getSources(), lookup); return null; } - private List> getAll ( List sources){ + private List> getAll ( List sources, ReferenceDefinitionSearchLookup lookup){ List> results = new LinkedList<>(); if (sources == null || sources.isEmpty()) { @@ -305,19 +305,29 @@ public class ReferenceService { } sources.sort(Comparator.comparing(ReferenceTypeSourceBaseConfigurationEntity::getOrdinal)); - List apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList(); + List apiSources = sources.stream().filter(x-> ReferenceTypeSourceType.API.equals(x.getType())).map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList(); apiSources.forEach(source -> { try { String auth = null; if (source.getAuth()!= null) { //auth = this.getAuthentication(source.getAuth()); } - results.addAll(getAllResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), null, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth)); + results.addAll(getAllApiResultsFromUrl(source.getUrl(), null, source.getResults(), source.getPaginationPath(), lookup, source.getLabel(), source.getKey(), source.getContentType(), source.getFirstPage(), source.getRequestBody(), source.getHttpMethod(), source.getFilterType(), source.getQueries(), auth)); } catch (Exception e) { logger.error(e.getLocalizedMessage(), e); } }); + List staticSources = sources.stream().filter(x-> ReferenceTypeSourceType.STATIC.equals(x.getType())).map(x-> (ReferenceTypeSourceStaticOptionConfigurationEntity)x).toList(); + staticSources.forEach(source -> { + Map map = new HashMap<>(); + source.getOptions().forEach(option -> { + map.put(option.getCode(), option.getValue()); + map.put("label", source.getLabel()); + map.put("key", source.getKey()); + }); + results.add(map); + }); return results; } @@ -349,23 +359,21 @@ public class ReferenceService { return finalBodyString; } - private List> getAllResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity jsonResultsPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List queries, String auth) throws Exception { + private List> getAllApiResultsFromUrl(String urlPath, FetchStrategy fetchStrategy, final ResultsConfigurationEntity resultsEntity, final String jsonPaginationPath, ReferenceDefinitionSearchLookup lookup, String label, String key, String contentType, String firstPage, String requestBody, ReferenceTypeExternalApiHTTPMethodType requestType, String filterType, List queries, String auth) throws Exception { Set pages = new HashSet<>(); - //String replacedPath = replaceCriteriaOnUrl(urlPath, externalUrlCriteria, firstPage, queries); - String replacedUrlPath = urlPath; - //String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage, queries); - String replacedUrlBody = requestBody; + String replacedUrlPath = replaceLookupFields(urlPath, lookup, firstPage, queries); + String replacedUrlBody = replaceLookupFields(requestBody, lookup, firstPage, queries); - Results results = getResultsFromUrl(replacedUrlPath, jsonResultsPath, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth); + Results results = getResultsFromUrl(replacedUrlPath, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth); if(results != null) { - if (filterType != null && filterType.equals("local") && (externalUrlCriteria.getLike() != null && !externalUrlCriteria.getLike().isEmpty())) { + if (filterType != null && filterType.equals("local") && (lookup.getLike() != null && !lookup.getLike().isEmpty())) { results.setResults(results.getResults().stream() - .filter(r -> r.get("name").toLowerCase().contains(externalUrlCriteria.getLike().toLowerCase())) + .filter(r -> r.get("name").toLowerCase().contains(lookup.getLike().toLowerCase())) .collect(Collectors.toList())); } if (fetchStrategy == FetchStrategy.FIRST) - return results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); + return results.getResults().stream().peek(x -> x.put("label", label)).peek(x -> x.put("key", key)).collect(Collectors.toList()); if (results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set for (int i = 2; i <= results.getPagination().get("pages"); i++) @@ -374,10 +382,10 @@ public class ReferenceService { //Long maxResults = configLoader.getExternalUrls().getMaxresults(); Long maxResults = Long.valueOf(1000); if ((maxResults > 0) && (results.getPagination().get("count") > maxResults)) - throw new HugeResultSet("The submitted search query " + externalUrlCriteria.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); + throw new HugeResultSet("The submitted search query " + lookup.getLike() + " is about to return " + results.getPagination().get("count") + " results... Please submit a more detailed search query"); Optional optionalResults = pages.parallelStream() - .map(page -> getResultsFromUrl(urlPath + "&page=" + page, jsonResultsPath, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth)) + .map(page -> getResultsFromUrl(urlPath + "&page=" + page, resultsEntity, jsonPaginationPath, contentType, replacedUrlBody, requestType, auth)) .filter(Objects::nonNull) .reduce((result1, result2) -> { result1.getResults().addAll(result2.getResults()); @@ -386,14 +394,52 @@ public class ReferenceService { Results remainingResults = optionalResults.orElseGet(Results::new); remainingResults.getResults().addAll(results.getResults()); - return remainingResults.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList()); + return remainingResults.getResults().stream().peek(x -> x.put("label", label)).peek(x -> x.put("key", key)).collect(Collectors.toList()); } else { return new LinkedList<>(); } } - protected Results getResultsFromUrl(String urlString, ResultsConfigurationEntity jsonResultsPath, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) { + private String replaceLookupFields(String urlPath, ReferenceDefinitionSearchLookup lookup, String firstPage, List queries){ + String completedPath = urlPath; + + if (urlPath.contains("{like}")){ + if (lookup.getLike() != null) { + completedPath = completedPath.replace("{like}", lookup.getLike()); + } else { + completedPath = completedPath.replace("{like}", "*"); + } + } + + if (urlPath.contains("{page}")){ + if (lookup.getPage() != null && lookup.getPage().getOffset() > 0) { + completedPath = completedPath.replace("{page}", String.valueOf(lookup.getPage().getOffset())); + } else { + completedPath = completedPath.replace("{page}", "1"); + } + } + + if (urlPath.contains("{pageSize}")){ + if (lookup.getPage() != null && lookup.getPage().getSize() > 0) { + completedPath = completedPath.replace("{pageSize}", String.valueOf(lookup.getPage().getSize())); + } else { + completedPath = completedPath.replace("{pageSize}", "100"); + } + } + + if (urlPath.contains("openaire")){ + if (lookup.getLike() != null) { + completedPath = completedPath.replace("{query}", lookup.getLike()); + } else { + completedPath = completedPath.replace("{query}", "*"); + } + } + + return completedPath; + } + + protected Results getResultsFromUrl(String urlString, ResultsConfigurationEntity resultsEntity, String jsonPaginationPath, String contentType, String requestBody, ReferenceTypeExternalApiHTTPMethodType httpMethod, String auth) { try { ResponseEntity response; @@ -412,7 +458,7 @@ public class ReferenceService { Results results = new Results(); if (response.getHeaders().get("Content-Type").get(0).contains("json")) { DocumentContext jsonContext = JsonPath.parse(response.getBody()); - results = this.getFromJson(jsonContext, jsonResultsPath); + results = this.getFromJson(jsonContext, resultsEntity); } @@ -430,24 +476,24 @@ public class ReferenceService { } - public static Results getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity jsonResultsPath) { - return new Results(parseData(jsonContext, jsonResultsPath), + public static Results getFromJson(DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { + return new Results(parseData(jsonContext, resultsEntity), new HashMap<>(1, 1)); } - private static List> parseData (DocumentContext jsonContext, ResultsConfigurationEntity jsonResultsPath) { - List > rawData = jsonContext.read(jsonResultsPath.getResultsArrayPath()); + private static List> parseData (DocumentContext jsonContext, ResultsConfigurationEntity resultsEntity) { + List > rawData = jsonContext.read(resultsEntity.getResultsArrayPath()); List> parsedData = new ArrayList<>(); rawData.forEach(stringObjectMap -> { Map map = new HashMap<>(); - jsonResultsPath.getFieldsMapping().forEach(field ->{ + resultsEntity.getFieldsMapping().forEach(field ->{ String pathValue = field.getResponsePath(); if (stringObjectMap.containsKey(pathValue)){ map.put(field.getCode(), stringObjectMap.get(pathValue)); - parsedData.add(map); } }); + parsedData.add(map); }); diff --git a/dmp-frontend/src/app/ui/admin/reference-type/editor/reference-type-editor.component.html b/dmp-frontend/src/app/ui/admin/reference-type/editor/reference-type-editor.component.html index 681cfbf66..1f621af05 100644 --- a/dmp-frontend/src/app/ui/admin/reference-type/editor/reference-type-editor.component.html +++ b/dmp-frontend/src/app/ui/admin/reference-type/editor/reference-type-editor.component.html @@ -132,6 +132,8 @@ + @@ -146,7 +148,7 @@
- +
diff --git a/dmp-frontend/src/app/ui/admin/reference-type/editor/reference-type-editor.component.ts b/dmp-frontend/src/app/ui/admin/reference-type/editor/reference-type-editor.component.ts index d206d0e5e..ec0baa33b 100644 --- a/dmp-frontend/src/app/ui/admin/reference-type/editor/reference-type-editor.component.ts +++ b/dmp-frontend/src/app/ui/admin/reference-type/editor/reference-type-editor.component.ts @@ -117,7 +117,9 @@ export class ReferenceTypeEditorComponent extends BaseEditor void) { @@ -225,14 +227,6 @@ export class ReferenceTypeEditorComponent extends BaseEditor 0){ - // for(let i =0; i < sourceSize; i++){ - // this.addFieldMapping(i); - // this.addOption(i); - // } - // } } @@ -247,21 +241,36 @@ export class ReferenceTypeEditorComponent extends BaseEditor 0){ + for(let i =0; i < sourceSize; i++){ + this.addFieldMapping(i, fieldssFormArray.at(i).get('code').value); + this.addOption(i, fieldssFormArray.at(i).get('code').value); + } + } + } + } + addSource(): void{ const source: ReferenceTypeSourceBaseConfigurationEditorModel = new ReferenceTypeSourceBaseConfigurationEditorModel(); (this.formGroup.get('definition').get('sources') as FormArray).push(source.buildForm()); const sourceIndex = (this.formGroup.get('definition').get('sources') as FormArray).length - 1; this.systemFieldsMapping.forEach(x => { - this.addFieldMapping(sourceIndex); - this.addOption(sourceIndex); + this.addFieldMapping(sourceIndex, null); + this.addOption(sourceIndex, null); }); const fieldsSize = (this.formGroup.get('definition').get('fields') as FormArray).length; if(fieldsSize && fieldsSize > 0){ for(let i =0; i < fieldsSize; i++){ - this.addFieldMapping(sourceIndex); - this.addOption(sourceIndex); + this.addFieldMapping(sourceIndex, null); + this.addOption(sourceIndex, null); } } } @@ -275,9 +284,23 @@ export class ReferenceTypeEditorComponent extends BaseEditor0){ + for(let i=0; i0){ + for(let i=0; i