MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

#8726 - adding openaire prefilling

This commit is contained in:
Bernaldo Mihasi 2023-07-14 13:47:01 +03:00
parent 1a15e1ddde
commit a8ce14af70
10 changed files with 192 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dmp-backend/web/src/main/java/eu/eudat/controllers/Prefillings.java
View File

@ -11,6 +11,7 @@ import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@RestController
@ -26,8 +27,8 @@ public class Prefillings {
}
@RequestMapping(method = RequestMethod.GET, value = {"/prefilling/list"}, produces = "application/json")
public ResponseEntity<ResponseItem<List<Prefilling>>> getPrefillingList(@RequestParam String like, @RequestParam String configId) {
List<Prefilling> prefillingList = prefillingManager.getPrefillings(like, configId);
public ResponseEntity<ResponseItem<List<Prefilling>>> getPrefillingList(@RequestParam String like) {
List<Prefilling> prefillingList = prefillingManager.getPrefillings(like);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<Prefilling>>().payload(prefillingList).status(ApiMessageCode.NO_MESSAGE));
}
@ -36,4 +37,10 @@ public class Prefillings {
DatasetWizardModel datasetWizardModel = prefillingManager.getPrefilledDataset(id, configId, profileId);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<DatasetWizardModel>().payload(datasetWizardModel).status(ApiMessageCode.NO_MESSAGE));
}
@RequestMapping(method = RequestMethod.POST, value = {"/prefilling/generateUsingData"}, produces = "application/json")
public ResponseEntity<ResponseItem<DatasetWizardModel>> getPrefillingDataset(@RequestBody Map<String, Object> data, @RequestParam String configId, @RequestParam UUID profileId) throws Exception {
DatasetWizardModel datasetWizardModel = prefillingManager.getPrefilledDatasetUsingData(data, configId, profileId);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<DatasetWizardModel>().payload(datasetWizardModel).status(ApiMessageCode.NO_MESSAGE));
}
}

35
dmp-backend/web/src/main/java/eu/eudat/logic/managers/PrefillingManager.java
View File

@ -1,5 +1,6 @@
package eu.eudat.logic.managers;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.entities.DatasetProfile;
import eu.eudat.logic.mapper.prefilling.PrefillingMapper;
@ -11,10 +12,7 @@ import eu.eudat.logic.services.ApiContext;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.prefilling.Prefilling;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.*;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@ -34,17 +32,29 @@ public class PrefillingManager {
public PrefillingManager(ApiContext apiContext, ConfigLoader configLoader, DatasetManager datasetManager, LicenseManager licenseManager) {
this.apiContext = apiContext;
this.configLoader = configLoader;
this.objectMapper = new ObjectMapper();
this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
this.datasetManager = datasetManager;
this.licenseManager = licenseManager;
}
public List<Prefilling> getPrefillings(String like, String configId) {
PrefillingConfig prefillingConfig = configLoader.getExternalUrls().getPrefillings().get(configId);
public List<Prefilling> getPrefillings(String like) {
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria();
externalUrlCriteria.setLike(like);
List<Map<String, String>> map = apiContext.getOperationsContext().getRemoteFetcher().getExternalGeneric(externalUrlCriteria, prefillingConfig.getPrefillingSearch());
return map.stream().map(submap -> objectMapper.convertValue(submap, Prefilling.class)).collect(Collectors.toList());
List<Prefilling> prefillings = new ArrayList<>();
List<Map<String, String>> map;
Map<String, PrefillingConfig> prefillingConfigs = configLoader.getExternalUrls().getPrefillings();
for (PrefillingConfig prefillingConfig: prefillingConfigs.values()) {
map = apiContext.getOperationsContext().getRemoteFetcher().getExternalGeneric(externalUrlCriteria, prefillingConfig.getPrefillingSearch());
prefillings.addAll(map.stream().map(submap -> objectMapper.convertValue(submap, Prefilling.class)).collect(Collectors.toList()));
if (prefillingConfig.getPrefillingSearch().getUrlConfig().isDataInListing()) {
List<Map<String, Object>> mapData = apiContext.getOperationsContext().getRemoteFetcher().getExternalGenericWithData(externalUrlCriteria, prefillingConfig.getPrefillingSearch());
for (int i = 0; i < mapData.size(); i++) {
prefillings.get(i).setData(mapData.get(i));
}
prefillings = prefillings.stream().filter(prefilling -> prefilling.getData() != null).collect(Collectors.toList());
}
}
return prefillings;
}
public DatasetWizardModel getPrefilledDataset(String prefillId, String configId, UUID profileId) throws Exception {
@ -55,6 +65,13 @@ public class PrefillingManager {
return PrefillingMapper.mapPrefilledEntityToDatasetWizard(prefillingEntity, prefillingGet, prefillingConfig.getType(), datasetProfile, datasetManager, licenseManager);
}
public DatasetWizardModel getPrefilledDatasetUsingData(Map<String, Object> data, String configId, UUID profileId) throws Exception {
PrefillingConfig prefillingConfig = configLoader.getExternalUrls().getPrefillings().get(configId);
PrefillingGet prefillingGet = prefillingConfig.getPrefillingGet();
DatasetProfile datasetProfile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(profileId);
return PrefillingMapper.mapPrefilledEntityToDatasetWizard(data, prefillingGet, prefillingConfig.getType(), datasetProfile, datasetManager, licenseManager);
}
private Map<String, Object> getSingle(String url, String id) {
RestTemplate restTemplate = new RestTemplate();
String parsedUrl = url.replace("{id}", id);

13
dmp-backend/web/src/main/java/eu/eudat/logic/mapper/prefilling/PrefillingMapper.java
View File

@ -107,7 +107,11 @@ public class PrefillingMapper {
//GK: Tags Special logic
if (parsedValue != null && !parsedValue.equals("null") && prefillingMapping.getTarget().equals("tags")) {
parsedValue = mapper.valueToTree(parseTags(parsedValue)).toString();
} else {
}
else if(!parsedValues.isEmpty() && prefillingMapping.getTarget().equals("tags")) {
parsedValue = mapper.valueToTree(parseTags(String.join(", ", parsedValues))).toString();
}
else {
parsedValue = mapper.valueToTree(parsedValue).toString();
}
setterMethod.invoke(datasetWizardModel, mapper.readValue(parsedValue, params[0]));
@ -155,7 +159,12 @@ public class PrefillingMapper {
}
break;
case TAGS:
properties.put(id, mapper.valueToTree(parseTags(parsedValue)).toString());
if(parsedValues.isEmpty()) {
properties.put(id, mapper.valueToTree(parseTags(parsedValue)).toString());
}
else {
properties.put(id, mapper.valueToTree(parseTags(String.join(", ", parsedValues))).toString());
}
break;
case DATASET_IDENTIFIER:
JSONObject datasetID = new JSONObject();

9
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/config/UrlConfiguration.java
View File

@ -11,6 +11,7 @@ public class UrlConfiguration {
private String label;
private Integer ordinal;
private String url;
private boolean dataInListing;
private DataUrlConfiguration data;
private String type;
private String paginationPath;
@ -47,6 +48,14 @@ public class UrlConfiguration {
this.url = url;
}
public boolean isDataInListing() {
return dataInListing;
}
@XmlElement(name = "dataInListing")
public void setDataInListing(boolean dataInListing) {
this.dataInListing = dataInListing;
}
public Integer getOrdinal() {
return ordinal;
}

59
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -179,6 +179,10 @@ public class RemoteFetcher {
return getAll(urlConfigurations, fetchStrategy, externalUrlCriteria);
}
public List<Map<String, Object>> getExternalGenericWithData(ExternalUrlCriteria externalUrlCriteria, GenericUrls genericUrls) {
List<UrlConfiguration> urlConfigurations = genericUrls.getUrls();
return getAllWithData(urlConfigurations, externalUrlCriteria);
}
private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) {
@ -213,6 +217,29 @@ public class RemoteFetcher {
return results;
}
private List<Map<String, Object>> getAllWithData(List<UrlConfiguration> urlConfigs, ExternalUrlCriteria externalUrlCriteria) {
List<Map<String, Object>> results = new LinkedList<>();
if (urlConfigs == null || urlConfigs.isEmpty()) {
return results;
}
urlConfigs.sort(Comparator.comparing(UrlConfiguration::getOrdinal));
urlConfigs.forEach(urlConfiguration -> {
ifFunderQueryExist(urlConfiguration, externalUrlCriteria);
if (urlConfiguration.getType() == null || urlConfiguration.getType().equals("External")) {
try {
results.addAll(getAllResultsFromUrlWithData(urlConfiguration.getUrl(), urlConfiguration.getData(), externalUrlCriteria, urlConfiguration.getContentType(), urlConfiguration.getFirstpage(), urlConfiguration.getRequestBody(), urlConfiguration.getRequestType(), urlConfiguration.getQueries()));
} catch (Exception e) {
logger.error(e.getLocalizedMessage(), e);
}
}
});
return results;
}
private void ifFunderQueryExist(UrlConfiguration urlConfiguration, ExternalUrlCriteria externalUrlCriteria) {
if (urlConfiguration.getFunderQuery() != null) {
if (externalUrlCriteria.getFunderId() != null && !urlConfiguration.getFunderQuery().startsWith("dmp:")) {
@ -343,6 +370,38 @@ public class RemoteFetcher {
}
}
private List<Map<String, Object>> getAllResultsFromUrlWithData(String path, final DataUrlConfiguration jsonDataPath, ExternalUrlCriteria externalUrlCriteria, String contentType, String firstPage, String requestBody, String requestType, List<QueryConfig> queries) {
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage, queries);
String replacedBody = replaceCriteriaOnUrl(requestBody, externalUrlCriteria, firstPage, queries);
try {
RestTemplate restTemplate = new RestTemplate();
HttpHeaders headers = new HttpHeaders();
HttpEntity<JsonNode> entity;
ResponseEntity<String> response;
if (contentType != null && !contentType.isEmpty()) {
headers.setAccept(Collections.singletonList(MediaType.valueOf(contentType)));
headers.setContentType(MediaType.valueOf(contentType));
}
JsonNode jsonBody = new ObjectMapper().readTree(replacedBody);
entity = new HttpEntity<>(jsonBody, headers);
response = restTemplate.exchange(replacedPath, HttpMethod.resolve(requestType), entity, String.class);
if (response.getStatusCode() == HttpStatus.OK) {
if (response.getHeaders().get("Content-Type").get(0).contains("json")) {
DocumentContext jsonContext = JsonPath.parse(response.getBody());
return jsonContext.read(jsonDataPath.getPath());
}
}
}
catch (Exception exception) {
logger.error(exception.getMessage(), exception);
}
return new LinkedList<>();
}
protected Results getResultsFromUrl(String urlString, DataUrlConfiguration jsonDataPath, String jsonPaginationPath, String contentType, String requestBody, String requestType) {

11
dmp-backend/web/src/main/java/eu/eudat/models/data/prefilling/Prefilling.java
View File

@ -1,8 +1,11 @@
package eu.eudat.models.data.prefilling;
import java.util.Map;
public class Prefilling {
private String pid;
private String name;
private Map<String, Object> data;
private String tag;
public String getPid() {
@ -21,6 +24,14 @@ public class Prefilling {
this.name = name;
}
public Map<String, Object> getData() {
return data;
}
public void setData(Map<String, Object> data) {
this.data = data;
}
public String getTag() {
return tag;
}

48
dmp-backend/web/src/main/resources/externalUrls/ExternalUrls.xml
View File

@ -1203,10 +1203,11 @@ but not
<prefillingSearch>
<urlConfig>
<key>zenodo</key>
<label>Zenodo</label>
<label>zenodo</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://zenodo.org/api/records/?page={page}&amp;size={pageSize}&amp;q=title:"{like}" doi:"{like}" conceptdoi:"{like}"</url>
<dataInListing>false</dataInListing>
<firstPage>1</firstPage>
<contenttype>application/json</contenttype>
<data>
@ -1259,6 +1260,51 @@ but not
</fixedMappings>
</prefillingGet>
</config>
<config id="openaire" type="openaire">
<prefillingSearch>
<urlConfig>
<key>openaire</key>
<label>openaire</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://services.openaire.eu/search/v2/api/datasets/?q={like}&amp;page={page}&amp;size={pageSize}&amp;format=json</url>
<dataInListing>true</dataInListing>
<firstPage>0</firstPage>
<contenttype>application/json</contenttype>
<data>
<path>$['results'][*]['result']['metadata']['oaf:entity']['oaf:result']</path>
<fields>
<id>'originalId'</id>
<name>'title'</name>
<description>'description'</description>
</fields>
</data>
<paginationpath>$['meta']['pagination']['page','pages','count']</paginationpath>
</urlConfig>
</prefillingSearch>
<prefillingGet>
<url></url>
<mappings>
<mapping source="title.content" target="label"/>
<mapping source="title.content" semanticTarget="rda.dataset.distribution.title"/>
<mapping source="description" target="description"/>
<mapping source="description" semanticTarget="rda.dataset.distribution.description"/>
<mapping source="children.instance.license" subSource="license" semanticTarget="rda.dataset.distribution.license.license_ref" />
<mapping source="subject.content" subSource="content" target="tags"/>
<mapping source="subject.content" subSource="content" semanticTarget="rda.dataset.keyword"/>
<mapping source="publisher" semanticTarget="rda.dataset.distribution.host.title"/>
<mapping source="language.classid" semanticTarget="rda.dataset.metadata.language"/>
<mapping source="issued.content" semanticTarget="rda.dataset.issued"/>
<mapping source="bestaccessright.classid" semanticTarget="rda.dataset.distribution.data_access"/>
<mapping source="pid.content" subSource="content" semanticTarget="rda.dataset.dataset_id"/>
<mapping source="pid.content" subSource="content" semanticTarget="rda.dataset.dataset_id.identifier"/>
<mapping source="pid.classid" subSource="classid" semanticTarget="rda.dataset.dataset_id.type"/>
</mappings>
<fixedMappings>
</fixedMappings>
</prefillingGet>
</config>
</prefillings>

1
dmp-frontend/src/app/core/model/dataset/prefilling.ts
View File

@ -1,5 +1,6 @@
export interface Prefilling {
pid: string;
name: string;
data: any;
tag: string;
}

8
dmp-frontend/src/app/core/services/prefilling.service.ts
View File

@ -15,11 +15,15 @@ export class PrefillingService {
this.actionUrl = configurationService.server + 'prefilling/';
}
public getPrefillingList(like: string, configId: string): Observable<Prefilling[]> {
return this.http.get<Prefilling[]>(this.actionUrl + 'list?configId=' + encodeURIComponent(configId) + '&like=' + encodeURIComponent(like), { headers: this.headers });
public getPrefillingList(like: string): Observable<Prefilling[]> {
return this.http.get<Prefilling[]>(this.actionUrl + 'list?like=' + encodeURIComponent(like), { headers: this.headers });
}
public getPrefillingDataset(pid: string, profileId: string, configId: string): Observable<DatasetWizardModel> {
return this.http.get<DatasetWizardModel>(this.actionUrl + '/generate/' + encodeURIComponent(pid) + '?configId=' + encodeURIComponent(configId) + '&profileId=' + encodeURIComponent(profileId), { headers: this.headers });
}
public getPrefillingDatasetUsingData(data: any, profileId: string, configId: string): Observable<DatasetWizardModel> {
return this.http.post<DatasetWizardModel>(this.actionUrl + '/generateUsingData' + '?configId=' + encodeURIComponent(configId) + '&profileId=' + encodeURIComponent(profileId), data, { headers: this.headers });
}
}

19
dmp-frontend/src/app/ui/dataset/dataset-wizard/prefill-dataset/prefill-dataset.component.ts
View File

@ -18,7 +18,6 @@ export class PrefillDatasetComponent extends BaseComponent implements OnInit {
progressIndication = false;
prefillAutoCompleteConfiguration: SingleAutoCompleteConfiguration;
configId: string = "zenodo";
isPrefilled: boolean = false;
prefillForm: FormGroup;
@ -56,7 +55,7 @@ export class PrefillDatasetComponent extends BaseComponent implements OnInit {
}
searchDatasets(query: string): Observable<Prefilling[]> {
return this.prefillingService.getPrefillingList(query, this.configId).pipe(map(prefilling => prefilling.sort((a, b) => {
return this.prefillingService.getPrefillingList(query).pipe(map(prefilling => prefilling.sort((a, b) => {
if(a.name > b.name) {
return 1;
} else if(a.name < b.name) {
@ -69,10 +68,18 @@ export class PrefillDatasetComponent extends BaseComponent implements OnInit {
next() {
if(this.isPrefilled) {
this.prefillingService.getPrefillingDataset(this.prefillForm.get('prefill').value.pid, this.prefillForm.get('profile').value.id, this.configId).subscribe(wizard => {
wizard.profile = this.prefillForm.get('profile').value;
this.closeDialog(wizard);
})
if(this.prefillForm.get('prefill').value.data == null) {
this.prefillingService.getPrefillingDataset(this.prefillForm.get('prefill').value.pid, this.prefillForm.get('profile').value.id, this.prefillForm.get('prefill').value.tag).subscribe(wizard => {
wizard.profile = this.prefillForm.get('profile').value;
this.closeDialog(wizard);
});
}
else {
this.prefillingService.getPrefillingDatasetUsingData(this.prefillForm.get('prefill').value.data, this.prefillForm.get('profile').value.id, this.prefillForm.get('prefill').value.tag).subscribe(wizard => {
wizard.profile = this.prefillForm.get('profile').value;
this.closeDialog(wizard);
});
}
} else {
this.closeDialog();
}