Add prefilling backend endpoints

This commit is contained in:
George Kalampokis 2021-12-10 15:37:39 +02:00
parent 4425a0bb5d
commit 728ea9a24f
25 changed files with 439 additions and 14 deletions

View File

@ -0,0 +1,39 @@
package eu.eudat.controllers;
import eu.eudat.logic.managers.PrefillingManager;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.helpers.responses.ResponseItem;
import eu.eudat.models.data.prefilling.Prefilling;
import eu.eudat.types.ApiMessageCode;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.UUID;
@RestController
@CrossOrigin
@RequestMapping(value = {"/api"})
public class Prefillings {
private final PrefillingManager prefillingManager;
@Autowired
public Prefillings(PrefillingManager prefillingManager) {
this.prefillingManager = prefillingManager;
}
@RequestMapping(method = RequestMethod.GET, value = {"/prefilling/list"}, produces = "application/json")
public ResponseEntity<ResponseItem<List<Prefilling>>> getPrefillingList(@RequestParam String like, @RequestParam String configId) {
List<Prefilling> prefillingList = prefillingManager.getPrefillings(like, configId);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<List<Prefilling>>().payload(prefillingList).status(ApiMessageCode.NO_MESSAGE));
}
@RequestMapping(method = RequestMethod.GET, value = {"/prefilling/get/{id}"}, produces = "application/json")
public ResponseEntity<ResponseItem<DatasetWizardModel>> getPrefillingDataset(@PathVariable String id, @RequestParam String configId, @RequestParam UUID profileId) throws Exception {
DatasetWizardModel datasetWizardModel = prefillingManager.getPrefilledDataset(id, configId, profileId);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<DatasetWizardModel>().payload(datasetWizardModel).status(ApiMessageCode.NO_MESSAGE));
}
}

View File

@ -0,0 +1,67 @@
package eu.eudat.logic.managers;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.entities.DatasetProfile;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.proxy.config.entities.PrefillingConfig;
import eu.eudat.logic.proxy.config.entities.PrefillingGet;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.models.data.datasetprofile.DatasetProfileOverviewModel;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.prefilling.Prefilling;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class PrefillingManager {
private final ApiContext apiContext;
private final ConfigLoader configLoader;
private final ObjectMapper objectMapper;
private final DatasetManager datasetManager;
private final DatasetProfileManager datasetProfileManager;
@Autowired
public PrefillingManager(ApiContext apiContext, ConfigLoader configLoader, DatasetManager datasetManager, DatasetProfileManager datasetProfileManager) {
this.apiContext = apiContext;
this.configLoader = configLoader;
this.objectMapper = new ObjectMapper();
this.datasetManager = datasetManager;
this.datasetProfileManager = datasetProfileManager;
}
public List<Prefilling> getPrefillings(String like, String configId) {
PrefillingConfig prefillingConfig = configLoader.getExternalUrls().getPrefillings().get(configId);
ExternalUrlCriteria externalUrlCriteria = new ExternalUrlCriteria();
externalUrlCriteria.setLike(like);
List<Map<String, String>> map = apiContext.getOperationsContext().getRemoteFetcher().getExternalGeneric(externalUrlCriteria, prefillingConfig.getPrefillingSearch());
return map.stream().map(submap -> objectMapper.convertValue(submap, Prefilling.class)).collect(Collectors.toList());
}
public DatasetWizardModel getPrefilledDataset(String prefillId, String configId, UUID profileId) throws Exception {
PrefillingConfig prefillingConfig = configLoader.getExternalUrls().getPrefillings().get(configId);
PrefillingGet prefillingGet = prefillingConfig.getPrefillingGet();
Map<String, Object> prefillingEntity = getSingle(prefillingGet.getUrl(), prefillId);
DatasetProfile datasetProfile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(profileId);
return DatasetWizardModel.fromPrefilledEntity(prefillingEntity, prefillingGet.getMappings(), datasetProfile, datasetManager);
}
private Map<String, Object> getSingle(String url, String id) {
RestTemplate restTemplate = new RestTemplate();
String parsedUrl = url.replace("{id}", id);
HttpHeaders headers = new HttpHeaders();
headers.setAccept(Collections.singletonList(MediaType.APPLICATION_JSON));
HttpEntity<String> entity = new HttpEntity("", headers);
return restTemplate.exchange(parsedUrl, HttpMethod.GET, entity, LinkedHashMap.class).getBody();
}
}

View File

@ -2,10 +2,14 @@ package eu.eudat.logic.proxy.config;
import eu.eudat.logic.proxy.config.entities.*;
import eu.eudat.logic.proxy.config.prefilling.PrefillingConfigMapAdapter;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
import java.io.Serializable;
import java.util.Map;
@XmlRootElement
public class ExternalUrls implements Serializable {
@ -26,6 +30,7 @@ public class ExternalUrls implements Serializable {
FunderUrls funders;
LicenseUrls licenses;
ValidationUrls validations;
Map<String, PrefillingConfig> prefillings;
public RegistryUrls getRegistries() {
@ -153,6 +158,16 @@ public class ExternalUrls implements Serializable {
public void setValidations(ValidationUrls validations) {
this.validations = validations;
}
public Map<String, PrefillingConfig> getPrefillings() {
return prefillings;
}
@XmlJavaTypeAdapter(PrefillingConfigMapAdapter.class)
@XmlElement(name = "prefillings")
public void setPrefillings(Map<String, PrefillingConfig> prefillings) {
this.prefillings = prefillings;
}
}

View File

@ -8,7 +8,7 @@ import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class DatasetUrls {
public class DatasetUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class FunderUrls {
public class FunderUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -0,0 +1,12 @@
package eu.eudat.logic.proxy.config.entities;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
import java.util.List;
public abstract class GenericUrls {
public abstract List<UrlConfiguration> getUrls();
public abstract FetchStrategy getFetchMode();
}

View File

@ -8,7 +8,7 @@ import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class GrantUrls {
public class GrantUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -8,7 +8,7 @@ import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class LicenseUrls {
public class LicenseUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class OrganisationUrls {
public class OrganisationUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -0,0 +1,29 @@
package eu.eudat.logic.proxy.config.entities;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "config")
public class PrefillingConfig {
private PrefillingSearch prefillingSearch;
private PrefillingGet prefillingGet;
public PrefillingSearch getPrefillingSearch() {
return prefillingSearch;
}
@XmlElement(name = "prefillingSearch")
public void setPrefillingSearch(PrefillingSearch prefillingSearch) {
this.prefillingSearch = prefillingSearch;
}
public PrefillingGet getPrefillingGet() {
return prefillingGet;
}
@XmlElement(name = "prefillingGet")
public void setPrefillingGet(PrefillingGet prefillingGet) {
this.prefillingGet = prefillingGet;
}
}

View File

@ -0,0 +1,32 @@
package eu.eudat.logic.proxy.config.entities;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class PrefillingGet{
private String url;
private List<PrefillingMapping> mappings;
public String getUrl() {
return url;
}
@XmlElement(name = "url")
public void setUrl(String url) {
this.url = url;
}
public List<PrefillingMapping> getMappings() {
return mappings;
}
@XmlElement(name = "mapping")
@XmlElementWrapper
public void setMappings(List<PrefillingMapping> mappings) {
this.mappings = mappings;
}
}

View File

@ -0,0 +1,38 @@
package eu.eudat.logic.proxy.config.entities;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "mapping")
public class PrefillingMapping {
private String source;
private String target;
private String maDmpTarget;
public String getSource() {
return source;
}
@XmlAttribute(name = "source")
public void setSource(String source) {
this.source = source;
}
public String getTarget() {
return target;
}
@XmlAttribute(name = "target")
public void setTarget(String target) {
this.target = target;
}
public String getMaDmpTarget() {
return maDmpTarget;
}
@XmlAttribute(name = "maDmpTarget")
public void setMaDmpTarget(String maDmpTarget) {
this.maDmpTarget = maDmpTarget;
}
}

View File

@ -0,0 +1,33 @@
package eu.eudat.logic.proxy.config.entities;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
import javax.xml.bind.annotation.XmlElement;
import java.util.ArrayList;
import java.util.List;
public class PrefillingSearch extends GenericUrls{
private UrlConfiguration urlConfig;
public UrlConfiguration getUrlConfig() {
return urlConfig;
}
@XmlElement(name = "urlConfig")
public void setUrlConfig(UrlConfiguration urlConfig) {
this.urlConfig = urlConfig;
}
@Override
public List<UrlConfiguration> getUrls() {
List<UrlConfiguration> urls = new ArrayList<>();
urls.add(urlConfig);
return urls;
}
@Override
public FetchStrategy getFetchMode() {
return null;
}
}

View File

@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ProjectUrls {
public class ProjectUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class RegistryUrls {
public class RegistryUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class RepositoryUrls {
public class RepositoryUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ResearcherUrls {
public class ResearcherUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElementWrapper;
import java.util.List;
public class ServiceUrls {
public class ServiceUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -10,7 +10,7 @@ import java.util.List;
/**
* Created by ikalyvas on 7/9/2018.
*/
public class TagUrls {
public class TagUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -8,7 +8,7 @@ import javax.xml.bind.annotation.XmlElementWrapper;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
public class ValidationUrls {
public class ValidationUrls extends GenericUrls{
List<UrlConfiguration> urls;
FetchStrategy fetchMode;

View File

@ -0,0 +1,35 @@
package eu.eudat.logic.proxy.config.prefilling;
import eu.eudat.logic.proxy.config.entities.PrefillingConfig;
import org.w3c.dom.Element;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.Unmarshaller;
import javax.xml.bind.annotation.adapters.XmlAdapter;
import java.util.HashMap;
import java.util.Map;
public class PrefillingConfigMapAdapter extends XmlAdapter<Object, Map<String, PrefillingConfig>> {
@Override
public Map<String, PrefillingConfig> unmarshal(Object v) throws Exception {
Map<String, PrefillingConfig> configMap = new HashMap<>();
Element element = (Element) v;
JAXBContext jaxbContext = JAXBContext.newInstance(PrefillingConfig.class);
Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
for (int i = 0; i < element.getChildNodes().getLength(); i++) {
if (element.getChildNodes().item(i).getAttributes() == null) {
continue;
}
String id = element.getChildNodes().item(i).getAttributes().getNamedItem("id").getNodeValue();
PrefillingConfig prefillingConfig = (PrefillingConfig) jaxbUnmarshaller.unmarshal(element.getChildNodes().item(i));
prefillingConfig = configMap.put(id, prefillingConfig);
System.out.println(prefillingConfig);
}
return configMap;
}
@Override
public Object marshal(Map<String, PrefillingConfig> v) throws Exception {
return null;
}
}

View File

@ -10,6 +10,7 @@ import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.FetchStrategy;
import eu.eudat.logic.proxy.config.UrlConfiguration;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.proxy.config.entities.GenericUrls;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.proxy.fetching.entities.Results;
@ -143,6 +144,12 @@ public class RemoteFetcher {
return data.size();
}
public List<Map<String, String>> getExternalGeneric(ExternalUrlCriteria externalUrlCriteria, GenericUrls genericUrls) {
List<UrlConfiguration> urlConfigurations = genericUrls.getUrls();
FetchStrategy fetchStrategy = genericUrls.getFetchMode();
return getAll(urlConfigurations, fetchStrategy, externalUrlCriteria);
}
private List<Map<String, String>> getAll(List<UrlConfiguration> urlConfigs, FetchStrategy fetchStrategy, ExternalUrlCriteria externalUrlCriteria) {
@ -309,8 +316,10 @@ public class RemoteFetcher {
results = RemoteFetcherUtils.getFromJsonWithFirstAndLastName(jsonContext, jsonDataPath);
} else {
results = new Results(jsonContext.read(jsonDataPath.getPath()
+ "[" + jsonDataPath.getFieldsUrlConfiguration().getName() + "," + jsonDataPath.getFieldsUrlConfiguration().getDescription()
+ "," + jsonDataPath.getFieldsUrlConfiguration().getUri() + "," + jsonDataPath.getFieldsUrlConfiguration().getId() + "]"),
+ "[" + (jsonDataPath.getFieldsUrlConfiguration().getName() != null ? jsonDataPath.getFieldsUrlConfiguration().getName(): "")
+ (jsonDataPath.getFieldsUrlConfiguration().getDescription() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getDescription(): "")
+ (jsonDataPath.getFieldsUrlConfiguration().getUri() !=null ? "," + jsonDataPath.getFieldsUrlConfiguration().getUri() : "")
+ (jsonDataPath.getFieldsUrlConfiguration().getId() != null ? "," + jsonDataPath.getFieldsUrlConfiguration().getId(): "") + "]"),
new HashMap<>(1, 1));
}
results.setResults(results.getResults().stream().map(e -> e.entrySet().stream().collect(Collectors.toMap(x -> this.transformKey(jsonDataPath,x.getKey()), Map.Entry::getValue)))
@ -370,6 +379,10 @@ public class RemoteFetcher {
results = new Results(values, new HashMap<>(1, 1));
}
if (results.getPagination().size() == 0) {
results.getPagination().put("pages", 1);
results.getPagination().put("count", results.getResults().size());
}
return results;
}
} catch (Exception exception) {

View File

@ -1,7 +1,12 @@
package eu.eudat.models.data.datasetwizard;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.entities.*;
import eu.eudat.elastic.entities.Tag;
import eu.eudat.logic.managers.DatasetManager;
import eu.eudat.logic.proxy.config.entities.PrefillingMapping;
import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.DataModel;
import eu.eudat.models.data.dataset.DataRepository;
import eu.eudat.models.data.dataset.Registry;
@ -12,7 +17,10 @@ import eu.eudat.models.data.externaldataset.ExternalDatasetListingModel;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import net.minidev.json.JSONValue;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.*;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
@ -297,6 +305,46 @@ public class DatasetWizardModel implements DataModel<Dataset, DatasetWizardModel
return entity;
}
public static DatasetWizardModel fromPrefilledEntity(Map<String, Object> prefilledEntity, List<PrefillingMapping> mappings,
DatasetProfile profile, DatasetManager datasetManager) throws Exception {
DatasetWizardModel datasetWizardModel = new DatasetWizardModel();
datasetWizardModel.setProfile(new DatasetProfileOverviewModel().fromDataModel(profile));
Dataset dataset = new Dataset();
dataset.setProfile(profile);
Map<String, Object> properties = new HashMap<>();
ObjectMapper objectMapper = new ObjectMapper();
JsonNode parentNode = objectMapper.readTree(objectMapper.writeValueAsString(datasetManager.getPagedProfile(datasetWizardModel, dataset)));
for (PrefillingMapping prefillingMapping: mappings) {
List<String> sourceKeys = Arrays.asList(prefillingMapping.getSource().split("\\."));
Object sourceValue = null;
for (String sourceKey: sourceKeys) {
if (sourceValue == null) {
sourceValue = prefilledEntity.get(sourceKey);
} else if (sourceValue instanceof Map) {
sourceValue = ((Map)sourceValue).get(sourceKey);
}
}
if (prefillingMapping.getTarget() != null) {
try {
String methodName = "set" + prefillingMapping.getTarget().substring(0, 1).toUpperCase(Locale.ROOT) + prefillingMapping.getTarget().substring(1);
Method setterMethod = DatasetWizardModel.class.getMethod(methodName, String.class);
setterMethod.invoke(datasetWizardModel, sourceValue);
}catch (NoSuchMethodException | InvocationTargetException | IllegalAccessException e) {
e.printStackTrace();
}
} else {
List<JsonNode> nodes = JsonSearcher.findNodes(parentNode, "rdaProperty", prefillingMapping.getMaDmpTarget());
for (JsonNode node: nodes) {
String id = node.get(0) != null ? node.get(0).get("id").asText() : node.get("id").asText();
properties.put(id, sourceValue);
}
}
}
dataset.setProperties(objectMapper.writeValueAsString(properties));
datasetWizardModel.setDatasetProfileDefinition(datasetManager.getPagedProfile(datasetWizardModel, dataset));
return datasetWizardModel;
}
@Override
public String getHint() {
return "datasetWizardModel";

View File

@ -0,0 +1,31 @@
package eu.eudat.models.data.prefilling;
public class Prefilling {
private String pid;
private String name;
private String tag;
public String getPid() {
return pid;
}
public void setPid(String pid) {
this.pid = pid;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getTag() {
return tag;
}
public void setTag(String tag) {
this.tag = tag;
}
}

View File

@ -1005,6 +1005,39 @@
<fetchMode>FIRST</fetchMode>
</validators>
<prefillings>
<config id="zenodo">
<prefillingSearch>
<urlConfig>
<key>zenodo</key>
<label>Zenodo</label>
<ordinal>1</ordinal>
<type>External</type>
<url>https://sandbox.zenodo.org/api/records/?page={page}&amp;size={pageSize}&amp;q="{like}"</url>
<firstPage>1</firstPage>
<contenttype>application/json</contenttype>
<data>
<path>$[*]</path>
<fields>
<id>'id'</id>
<name>'title'</name>
<description>'description'</description>
</fields>
</data>
<paginationpath>$['hits']['total']</paginationpath>
</urlConfig>
</prefillingSearch>
<prefillingGet>
<url>https://sandbox.zenodo.org/api/records/{id}</url>
<mappings>
<mapping source="metadata.title" target="label" />
<mapping source="metadata.description" target="description" />
<mapping source="metadata.license.id" maDmpTarget="dataset.distribution.license.license_ref" />
</mappings>
</prefillingGet>
</config>
</prefillings>
</externalUrls>