From a8ce14af700a036efbc54a8abb5f00d81037db2b Mon Sep 17 00:00:00 2001 From: Aldo Mihasi Date: Fri, 14 Jul 2023 13:47:01 +0300 Subject: [PATCH] #8726 - adding openaire prefilling --- .../eu/eudat/controllers/Prefillings.java | 11 +++- .../logic/managers/PrefillingManager.java | 35 ++++++++--- .../mapper/prefilling/PrefillingMapper.java | 13 +++- .../logic/proxy/config/UrlConfiguration.java | 9 +++ .../logic/proxy/fetching/RemoteFetcher.java | 59 +++++++++++++++++++ .../models/data/prefilling/Prefilling.java | 11 ++++ .../resources/externalUrls/ExternalUrls.xml | 48 ++++++++++++++- .../src/app/core/model/dataset/prefilling.ts | 1 + .../app/core/services/prefilling.service.ts | 8 ++- .../prefill-dataset.component.ts | 19 ++++-- 10 files changed, 192 insertions(+), 22 deletions(-) diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/Prefillings.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/Prefillings.java index c6c3335d6..7d39c2630 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/Prefillings.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/Prefillings.java @@ -11,6 +11,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; import java.util.List; +import java.util.Map; import java.util.UUID; @RestController @@ -26,8 +27,8 @@ public class Prefillings { } @RequestMapping(method = RequestMethod.GET, value = {"/prefilling/list"}, produces = "application/json") - public ResponseEntity>> getPrefillingList(@RequestParam String like, @RequestParam String configId) { - List prefillingList = prefillingManager.getPrefillings(like, configId); + public ResponseEntity>> getPrefillingList(@RequestParam String like) { + List prefillingList = prefillingManager.getPrefillings(like); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem>().payload(prefillingList).status(ApiMessageCode.NO_MESSAGE)); } @@ -36,4 +37,10 @@ public class Prefillings { DatasetWizardModel datasetWizardModel = prefillingManager.getPrefilledDataset(id, configId, profileId); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(datasetWizardModel).status(ApiMessageCode.NO_MESSAGE)); } + + @RequestMapping(method = RequestMethod.POST, value = {"/prefilling/generateUsingData"}, produces = "application/json") + public ResponseEntity> getPrefillingDataset(@RequestBody Map data, @RequestParam String configId, @RequestParam UUID profileId) throws Exception { + DatasetWizardModel datasetWizardModel = prefillingManager.getPrefilledDatasetUsingData(data, configId, profileId); + return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(datasetWizardModel).status(ApiMessageCode.NO_MESSAGE)); + } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java index 81d5fe3e0..e234ff300 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java @@ -1,5 +1,6 @@ package eu.eudat.logic.managers; +import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.data.entities.DatasetProfile; import eu.eudat.logic.mapper.prefilling.PrefillingMapper; @@ -11,10 +12,7 @@ import eu.eudat.logic.services.ApiContext; import eu.eudat.models.data.datasetwizard.DatasetWizardModel; import eu.eudat.models.data.prefilling.Prefilling; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.HttpEntity; -import org.springframework.http.HttpHeaders; -import org.springframework.http.HttpMethod; -import org.springframework.http.MediaType; +import org.springframework.http.*; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; @@ -34,17 +32,29 @@ public class PrefillingManager { public PrefillingManager(ApiContext apiContext, ConfigLoader configLoader, DatasetManager datasetManager, LicenseManager licenseManager) { this.apiContext = apiContext; this.configLoader = configLoader; - this.objectMapper = new ObjectMapper(); + this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); this.datasetManager = datasetManager; this.licenseManager = licenseManager; } - public List getPrefillings(String like, String configId) { - PrefillingConfig prefillingConfig = configLoader.getExternalUrls().getPrefillings().get(configId); + public List getPrefillings(String like) { ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria(); externalUrlCriteria.setLike(like); - List> map = apiContext.getOperationsContext().getRemoteFetcher().getExternalGeneric(externalUrlCriteria, prefillingConfig.getPrefillingSearch()); - return map.stream().map(submap -> objectMapper.convertValue(submap, Prefilling.class)).collect(Collectors.toList()); + List prefillings = new ArrayList<>(); + List> map; + Map prefillingConfigs = configLoader.getExternalUrls().getPrefillings(); + for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) { + map = apiContext.getOperationsContext().getRemoteFetcher().getExternalGeneric(externalUrlCriteria, prefillingConfig.getPrefillingSearch()); + prefillings.addAll(map.stream().map(submap -> objectMapper.convertValue(submap, Prefilling.class)).collect(Collectors.toList())); + if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) { + List> mapData = apiContext.getOperationsContext().getRemoteFetcher().getExternalGenericWithData(externalUrlCriteria, prefillingConfig.getPrefillingSearch()); + for (int i = 0; i < mapData.size(); i++) { + prefillings.get(i).setData(mapData.get(i)); + } + prefillings = prefillings.stream().filter(prefilling -> prefilling.getData() != null).collect(Collectors.toList()); + } + } + return prefillings; } public DatasetWizardModel getPrefilledDataset(String prefillId, String configId, UUID profileId) throws Exception { @@ -55,6 +65,13 @@ public class PrefillingManager { return PrefillingMapper.mapPrefilledEntityToDatasetWizard(prefillingEntity, prefillingGet, prefillingConfig.getType(), datasetProfile, datasetManager, licenseManager); } + public DatasetWizardModel getPrefilledDatasetUsingData(Map data, String configId, UUID profileId) throws Exception { + PrefillingConfig prefillingConfig = configLoader.getExternalUrls().getPrefillings().get(configId); + PrefillingGet prefillingGet = prefillingConfig.getPrefillingGet(); + DatasetProfile datasetProfile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(profileId); + return PrefillingMapper.mapPrefilledEntityToDatasetWizard(data, prefillingGet, prefillingConfig.getType(), datasetProfile, datasetManager, licenseManager); + } + private Map getSingle(String url, String id) { RestTemplate restTemplate = new RestTemplate(); String parsedUrl = url.replace("{id}", id); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/mapper/prefilling/PrefillingMapper.java b/dmp-backend/web/src/main/java/eu/eudat/logic/mapper/prefilling/PrefillingMapper.java index a6b4d2ea3..4c3a6653b 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/mapper/prefilling/PrefillingMapper.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/mapper/prefilling/PrefillingMapper.java @@ -107,7 +107,11 @@ public class PrefillingMapper { //GK: Tags Special logic if (parsedValue != null && !parsedValue.equals("null") && prefillingMapping.getTarget().equals("tags")) { parsedValue = mapper.valueToTree(parseTags(parsedValue)).toString(); - } else { + } + else if(!parsedValues.isEmpty() && prefillingMapping.getTarget().equals("tags")) { + parsedValue = mapper.valueToTree(parseTags(String.join(", ", parsedValues))).toString(); + } + else { parsedValue = mapper.valueToTree(parsedValue).toString(); } setterMethod.invoke(datasetWizardModel, mapper.readValue(parsedValue, params[0])); @@ -155,7 +159,12 @@ public class PrefillingMapper { } break; case TAGS: - properties.put(id, mapper.valueToTree(parseTags(parsedValue)).toString()); + if(parsedValues.isEmpty()) { + properties.put(id, mapper.valueToTree(parseTags(parsedValue)).toString()); + } + else { + properties.put(id, mapper.valueToTree(parseTags(String.join(", ", parsedValues))).toString()); + } break; case DATASET_IDENTIFIER: JSONObject datasetID = new JSONObject(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java index 232d619d1..50f2093fa 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java @@ -11,6 +11,7 @@ public class UrlConfiguration { private String label; private Integer ordinal; private String url; + private boolean dataInListing; private DataUrlConfiguration data; private String type; private String paginationPath; @@ -47,6 +48,14 @@ public class UrlConfiguration { this.url = url; } + public boolean isDataInListing() { + return dataInListing; + } + @XmlElement(name = "dataInListing") + public void setDataInListing(boolean dataInListing) { + this.dataInListing = dataInListing; + } + public Integer getOrdinal() { return ordinal; } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index 77f05c1fc..c77964083 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -179,6 +179,10 @@ public class RemoteFetcher { return getAll(urlConfigurations, fetchStrategy, externalUrlCriteria); } + public List> getExternalGenericWithData(ExternalUrlCriteria externalUrlCriteria, GenericUrls genericUrls) { + List urlConfigurations = genericUrls.getUrls(); + return getAllWithData(urlConfigurations, externalUrlCriteria); + } private List> getAll(List urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) { @@ -213,6 +217,29 @@ public class RemoteFetcher { return results; } + private List> getAllWithData(List urlConfigs, ExternalUrlCriteria externalUrlCriteria) { + + List> results = new LinkedList<>(); + + if (urlConfigs == null || urlConfigs.isEmpty()) { + return results; + } + + urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal)); + urlConfigs.forEach(urlConfiguration -> { + ifFunderQueryExist(urlConfiguration, externalUrlCriteria); + if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) { + try { + results.addAll(getAllResultsFromUrlWithData(urlConfiguration.getUrl(), urlConfiguration.getData(), externalUrlCriteria, urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getQueries())); + } catch (Exception e) { + logger.error(e.getLocalizedMessage(), e); + } + } + }); + return results; + + } + private void ifFunderQueryExist(UrlConfiguration urlConfiguration, ExternalUrlCriteria externalUrlCriteria) { if (urlConfiguration.getFunderQuery() != null) { if (externalUrlCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) { @@ -343,6 +370,38 @@ public class RemoteFetcher { } } + private List> getAllResultsFromUrlWithData(String path, final DataUrlConfiguration jsonDataPath, ExternalUrlCriteria externalUrlCriteria, String contentType, String firstPage, String requestBody, String requestType, List queries) { + + String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage, queries); + String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage, queries); + + try { + RestTemplate restTemplate = new RestTemplate(); + HttpHeaders headers = new HttpHeaders(); + HttpEntity entity; + ResponseEntity response; + if (contentType != null && !contentType.isEmpty()) { + headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType))); + headers.setContentType(MediaType.valueOf(contentType)); + } + JsonNode jsonBody = new ObjectMapper().readTree(replacedBody); + entity = new HttpEntity<>(jsonBody, headers); + + response = restTemplate.exchange(replacedPath, HttpMethod.resolve(requestType), entity, String.class); + if (response.getStatusCode() == HttpStatus.OK) { + if (response.getHeaders().get("Content-Type").get(0).contains("json")) { + DocumentContext jsonContext = JsonPath.parse(response.getBody()); + return jsonContext.read(jsonDataPath.getPath()); + } + } + } + catch (Exception exception) { + logger.error(exception.getMessage(), exception); + } + + return new LinkedList<>(); + } + protected Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/prefilling/Prefilling.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/prefilling/Prefilling.java index 6c09e6b2e..67babe9de 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/prefilling/Prefilling.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/prefilling/Prefilling.java @@ -1,8 +1,11 @@ package eu.eudat.models.data.prefilling; +import java.util.Map; + public class Prefilling { private String pid; private String name; + private Map data; private String tag; public String getPid() { @@ -21,6 +24,14 @@ public class Prefilling { this.name = name; } + public Map getData() { + return data; + } + + public void setData(Map data) { + this.data = data; + } + public String getTag() { return tag; } diff --git a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml index bdedc03a5..95aaac7fa 100644 --- a/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml +++ b/dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml @@ -1203,10 +1203,11 @@ but not zenodo - + 1 External https://zenodo.org/api/records/?page={page}&size={pageSize}&q=title:"{like}" doi:"{like}" conceptdoi:"{like}" + false 1 application/json @@ -1259,6 +1260,51 @@ but not + + + + openaire + + 1 + External + https://services.openaire.eu/search/v2/api/datasets/?q={like}&page={page}&size={pageSize}&format=json + true + 0 + application/json + + $['results'][*]['result']['metadata']['oaf:entity']['oaf:result'] + + 'originalId' + 'title' + 'description' + + + $['meta']['pagination']['page','pages','count'] + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/dmp-frontend/src/app/core/model/dataset/prefilling.ts b/dmp-frontend/src/app/core/model/dataset/prefilling.ts index cc3e36a4b..1b5b93538 100644 --- a/dmp-frontend/src/app/core/model/dataset/prefilling.ts +++ b/dmp-frontend/src/app/core/model/dataset/prefilling.ts @@ -1,5 +1,6 @@ export interface Prefilling { pid: string; name: string; + data: any; tag: string; } diff --git a/dmp-frontend/src/app/core/services/prefilling.service.ts b/dmp-frontend/src/app/core/services/prefilling.service.ts index 3d5527303..fa940a1f6 100644 --- a/dmp-frontend/src/app/core/services/prefilling.service.ts +++ b/dmp-frontend/src/app/core/services/prefilling.service.ts @@ -15,11 +15,15 @@ export class PrefillingService { this.actionUrl = configurationService.server + 'prefilling/'; } - public getPrefillingList(like: string, configId: string): Observable { - return this.http.get(this.actionUrl + 'list?configId=' + encodeURIComponent(configId) + '&like=' + encodeURIComponent(like), { headers: this.headers }); + public getPrefillingList(like: string): Observable { + return this.http.get(this.actionUrl + 'list?like=' + encodeURIComponent(like), { headers: this.headers }); } public getPrefillingDataset(pid: string, profileId: string, configId: string): Observable { return this.http.get(this.actionUrl + '/generate/' + encodeURIComponent(pid) + '?configId=' + encodeURIComponent(configId) + '&profileId=' + encodeURIComponent(profileId), { headers: this.headers }); } + + public getPrefillingDatasetUsingData(data: any, profileId: string, configId: string): Observable { + return this.http.post(this.actionUrl + '/generateUsingData' + '?configId=' + encodeURIComponent(configId) + '&profileId=' + encodeURIComponent(profileId), data, { headers: this.headers }); + } } diff --git a/dmp-frontend/src/app/ui/dataset/dataset-wizard/prefill-dataset/prefill-dataset.component.ts b/dmp-frontend/src/app/ui/dataset/dataset-wizard/prefill-dataset/prefill-dataset.component.ts index 3d21e286e..e46a2b184 100644 --- a/dmp-frontend/src/app/ui/dataset/dataset-wizard/prefill-dataset/prefill-dataset.component.ts +++ b/dmp-frontend/src/app/ui/dataset/dataset-wizard/prefill-dataset/prefill-dataset.component.ts @@ -18,7 +18,6 @@ export class PrefillDatasetComponent extends BaseComponent implements OnInit { progressIndication = false; prefillAutoCompleteConfiguration: SingleAutoCompleteConfiguration; - configId: string = "zenodo"; isPrefilled: boolean = false; prefillForm: FormGroup; @@ -56,7 +55,7 @@ export class PrefillDatasetComponent extends BaseComponent implements OnInit { } searchDatasets(query: string): Observable { - return this.prefillingService.getPrefillingList(query, this.configId).pipe(map(prefilling => prefilling.sort((a, b) => { + return this.prefillingService.getPrefillingList(query).pipe(map(prefilling => prefilling.sort((a, b) => { if(a.name > b.name) { return 1; } else if(a.name < b.name) { @@ -69,10 +68,18 @@ export class PrefillDatasetComponent extends BaseComponent implements OnInit { next() { if(this.isPrefilled) { - this.prefillingService.getPrefillingDataset(this.prefillForm.get('prefill').value.pid, this.prefillForm.get('profile').value.id, this.configId).subscribe(wizard => { - wizard.profile = this.prefillForm.get('profile').value; - this.closeDialog(wizard); - }) + if(this.prefillForm.get('prefill').value.data == null) { + this.prefillingService.getPrefillingDataset(this.prefillForm.get('prefill').value.pid, this.prefillForm.get('profile').value.id, this.prefillForm.get('prefill').value.tag).subscribe(wizard => { + wizard.profile = this.prefillForm.get('profile').value; + this.closeDialog(wizard); + }); + } + else { + this.prefillingService.getPrefillingDatasetUsingData(this.prefillForm.get('prefill').value.data, this.prefillForm.get('profile').value.id, this.prefillForm.get('prefill').value.tag).subscribe(wizard => { + wizard.profile = this.prefillForm.get('profile').value; + this.closeDialog(wizard); + }); + } } else { this.closeDialog(); }