286 lines
16 KiB
Java
286 lines
16 KiB
Java
package eu.eudat.logic.managers;
|
|
|
|
import eu.eudat.commons.types.descriptiontemplate.FieldEntity;
|
|
import eu.eudat.commons.types.descriptiontemplate.fielddata.AutoCompleteDataEntity;
|
|
import eu.eudat.commons.types.xml.XmlBuilder;
|
|
import eu.eudat.data.DescriptionTemplateEntity;
|
|
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
|
|
import eu.eudat.service.remotefetcher.config.entities.GeneralUrls;
|
|
import eu.eudat.service.remotefetcher.RemoteFetcherService;
|
|
import eu.eudat.logic.services.ApiContext;
|
|
import eu.eudat.logic.services.operations.DatabaseRepository;
|
|
import eu.eudat.service.remotefetcher.models.ExternalAutocompleteFieldResult;
|
|
import eu.eudat.service.remotefetcher.config.AuthenticationConfiguration;
|
|
import eu.eudat.service.remotefetcher.config.DataFieldsUrlConfiguration;
|
|
import eu.eudat.service.remotefetcher.config.DataUrlConfiguration;
|
|
import eu.eudat.service.remotefetcher.config.UrlConfiguration;
|
|
import eu.eudat.service.remotefetcher.criteria.ExternalReferenceCriteria;
|
|
import eu.eudat.service.remotefetcher.criteria.FetchStrategy;
|
|
import org.slf4j.Logger;
|
|
import org.slf4j.LoggerFactory;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.http.MediaType;
|
|
import org.springframework.stereotype.Component;
|
|
import org.w3c.dom.Document;
|
|
import org.w3c.dom.Element;
|
|
import org.w3c.dom.Node;
|
|
import org.w3c.dom.NodeList;
|
|
|
|
import javax.management.InvalidApplicationException;
|
|
import javax.xml.transform.OutputKeys;
|
|
import javax.xml.transform.Transformer;
|
|
import javax.xml.transform.TransformerException;
|
|
import javax.xml.transform.TransformerFactory;
|
|
import javax.xml.transform.dom.DOMSource;
|
|
import javax.xml.transform.stream.StreamResult;
|
|
import javax.xml.xpath.*;
|
|
import java.io.StringWriter;
|
|
import java.net.URI;
|
|
import java.net.URISyntaxException;
|
|
import java.util.ArrayList;
|
|
import java.util.LinkedList;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
|
|
|
|
@Component
|
|
public class DatasetProfileManager {
|
|
private static final Logger logger = LoggerFactory.getLogger(DatasetProfileManager.class);
|
|
private static final List<String> cache = new ArrayList<>();
|
|
|
|
private final DatabaseRepository databaseRepository;
|
|
private final ConfigLoader configLoader;
|
|
private final RemoteFetcherService remoteFetcherService;
|
|
|
|
@Autowired
|
|
public DatasetProfileManager(ApiContext apiContext, ConfigLoader configLoader, RemoteFetcherService remoteFetcherService) {
|
|
this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
|
|
this.configLoader = configLoader;
|
|
this.remoteFetcherService = remoteFetcherService;
|
|
}
|
|
|
|
public FieldEntity queryForField(String xml, String fieldId) throws XPathExpressionException {
|
|
FieldEntity fieldEntity = new FieldEntity();
|
|
Document document = XmlBuilder.fromXml(xml);
|
|
XPathFactory xpathFactory = XPathFactory.newInstance();
|
|
XPath xpath = xpathFactory.newXPath();
|
|
XPathExpression expr =
|
|
xpath.compile("//field[@id='" + fieldId + "']");
|
|
Element name = (Element) expr.evaluate(document, XPathConstants.NODE);
|
|
fieldEntity.fromXml(name);
|
|
return fieldEntity;
|
|
}
|
|
|
|
public List<ExternalAutocompleteFieldResult> getAutocomplete(AutoCompleteDataEntity data, String like) {
|
|
/*List<ExternalAutocompleteFieldModel> result = new LinkedList<>();
|
|
SimpleClientHttpRequestFactory simpleFactory = new SimpleClientHttpRequestFactory();
|
|
|
|
RestTemplate restTemplate = new RestTemplate(simpleFactory);
|
|
HttpHeaders headers = new HttpHeaders();
|
|
DocumentContext jsonContext = null;
|
|
HttpEntity<String> entity;
|
|
ResponseEntity response;
|
|
List<Map<String, String>> jsonItems;
|
|
int i = 0;
|
|
for (AutoCompleteData.AutoCompleteSingleData singleData: data.getAutoCompleteSingleDataList()) {
|
|
switch (AutoCompleteData.AutocompleteType.fromValue(singleData.getAutocompleteType())) {
|
|
case UNCACHED:
|
|
|
|
String url = singleData.getUrl();
|
|
String mediaType = "";
|
|
if (url.contains("openaire") || url.contains("zenodo")) {
|
|
mediaType = "application/json; charset=utf-8";
|
|
if (url.contains("zenodo")) {
|
|
url = url.replace("?", "/?");
|
|
}
|
|
|
|
url = url.replace("{like}", like.equals("") ? "*" : like);
|
|
url = url.replace("%20", " ");
|
|
url = url.replace("%22", "\"");
|
|
url = url.replace("&", "&");
|
|
} else {
|
|
mediaType = "application/vnd.api+json; charset=utf-8";
|
|
url += "?search=" + like;
|
|
}
|
|
|
|
if (!url.contains("zenodo")) {
|
|
headers.setAccept(Collections.singletonList(MediaType.valueOf(mediaType)));
|
|
}
|
|
headers.setContentType(MediaType.APPLICATION_JSON);
|
|
entity = new HttpEntity<>("parameters", headers);
|
|
|
|
|
|
response = restTemplate.exchange(url, HttpMethod.GET, entity, Object.class);
|
|
jsonContext = JsonPath.parse(response.getBody());
|
|
jsonItems = jsonContext.read(singleData.getOptionsRoot() + "['" + singleData.getAutoCompleteOptions().getLabel() + "','" + singleData.getAutoCompleteOptions().getValue() + "','" + singleData.getAutoCompleteOptions().getSource() + "','" + "uri" + "']");
|
|
jsonItems.forEach(item -> result.add(new ExternalAutocompleteFieldModel(parseItem(item.get(singleData.getAutoCompleteOptions().getValue())), parseItem(item.get(singleData.getAutoCompleteOptions().getLabel())), item.get(singleData.getAutoCompleteOptions().getSource()) != null ? parseItem(item.get(singleData.getAutoCompleteOptions().getSource())) : singleData.getAutoCompleteOptions().getSource(), parseItem(item.get("uri")))));
|
|
break;
|
|
case CACHED:
|
|
headers.setAccept(Collections.singletonList(MediaType.valueOf("text/plain; charset=utf-8")));
|
|
headers.setContentType(MediaType.TEXT_PLAIN);
|
|
entity = new HttpEntity<>("parameters", headers);
|
|
|
|
if (cache.size() <= i) {
|
|
response = restTemplate.exchange(singleData.getUrl(), HttpMethod.GET, entity, String.class);
|
|
cache.add((String) response.getBody());
|
|
}
|
|
jsonContext = JsonPath.parse(cache.get(i));
|
|
jsonItems = jsonContext.read(singleData.getOptionsRoot() + "['" + singleData.getAutoCompleteOptions().getLabel() + "','" + singleData.getAutoCompleteOptions().getValue() + "','" + singleData.getAutoCompleteOptions().getSource() + "','" + "uri" + "']");
|
|
jsonItems.stream().filter(item -> item.get(singleData.getAutoCompleteOptions().getLabel()).toLowerCase().contains(like.toLowerCase()))
|
|
.forEach(item -> result.add(new ExternalAutocompleteFieldModel(item.get(singleData.getAutoCompleteOptions().getValue()), item.get(singleData.getAutoCompleteOptions().getLabel()), item.get(singleData.getAutoCompleteOptions().getSource()) != null ? item.get(singleData.getAutoCompleteOptions().getSource()) : singleData.getAutoCompleteOptions().getSource(), item.get("uri"))));
|
|
i++;
|
|
break;
|
|
}
|
|
|
|
}
|
|
|
|
return result.stream().sorted(Comparator.comparing(ExternalAutocompleteFieldModel::getLabel)).collect(Collectors.toList());
|
|
*/
|
|
List<ExternalAutocompleteFieldResult> result = new LinkedList<>();
|
|
ExternalReferenceCriteria urlCriteria = new ExternalReferenceCriteria();
|
|
GeneralUrls genericUrls = new GeneralUrls();
|
|
int ordinal = 1;
|
|
List<Map<String, String>> rawResults = new ArrayList<>();
|
|
genericUrls.setFetchMode(FetchStrategy.FIRST);
|
|
urlCriteria.setLike(like);
|
|
for (AutoCompleteDataEntity.AutoCompleteSingleData singleData : data.getAutoCompleteSingleDataList()) {
|
|
UrlConfiguration urlConfiguration = new UrlConfiguration();
|
|
try {
|
|
URI uri;
|
|
if (singleData.getUrl().contains("?")) {
|
|
uri = new URI(singleData.getUrl().substring(0, singleData.getUrl().lastIndexOf("?")));
|
|
} else {
|
|
uri = new URI(singleData.getUrl());
|
|
}
|
|
String source = singleData.getAutoCompleteOptions().getSource();
|
|
source = source != null && !source.isEmpty() ? source : uri.getHost();
|
|
String uriString = singleData.getAutoCompleteOptions().getUri();
|
|
uriString = uriString != null && !uriString.isEmpty() ? uriString : "uri";
|
|
String parsedUrl = singleData.getUrl();
|
|
parsedUrl = parsedUrl.replace("%20", " ");
|
|
parsedUrl = parsedUrl.replace("%22", "\"");
|
|
while (parsedUrl.contains("&")) {
|
|
parsedUrl = parsedUrl.replace("&", "&");
|
|
}
|
|
urlConfiguration.setUrl(parsedUrl);
|
|
urlConfiguration.setOrdinal(ordinal);
|
|
urlConfiguration.setType("External");
|
|
urlConfiguration.setContentType(MediaType.APPLICATION_JSON_VALUE);
|
|
urlConfiguration.setFirstpage("1");
|
|
urlConfiguration.setRequestType(singleData.getMethod() != null ? singleData.getMethod() : "GET");
|
|
DataUrlConfiguration dataUrlConfiguration = new DataUrlConfiguration();
|
|
dataUrlConfiguration.setPath(singleData.getOptionsRoot());
|
|
DataFieldsUrlConfiguration fieldsUrlConfiguration = new DataFieldsUrlConfiguration();
|
|
fieldsUrlConfiguration.setId(singleData.getAutoCompleteOptions().getValue());
|
|
fieldsUrlConfiguration.setName(singleData.getAutoCompleteOptions().getLabel());
|
|
fieldsUrlConfiguration.setSource(singleData.getAutoCompleteOptions().getSource().isEmpty()? null : singleData.getAutoCompleteOptions().getSource());
|
|
fieldsUrlConfiguration.setUri(uriString);
|
|
dataUrlConfiguration.setFieldsUrlConfiguration(fieldsUrlConfiguration);
|
|
urlConfiguration.setKey(source);
|
|
urlConfiguration.setLabel(source);
|
|
urlConfiguration.setData(dataUrlConfiguration);
|
|
if (singleData.getHasAuth()) {
|
|
AuthenticationConfiguration authenticationConfiguration = new AuthenticationConfiguration();
|
|
authenticationConfiguration.setAuthUrl(singleData.getAuth().getUrl());
|
|
authenticationConfiguration.setAuthMethod(singleData.getAuth().getMethod());
|
|
authenticationConfiguration.setAuthTokenPath(singleData.getAuth().getPath());
|
|
authenticationConfiguration.setAuthRequestBody(singleData.getAuth().getBody());
|
|
authenticationConfiguration.setType(singleData.getAuth().getType());
|
|
urlConfiguration.setAuth(authenticationConfiguration);
|
|
}
|
|
genericUrls.getUrls().add(urlConfiguration);
|
|
List<Map<String, String>> singleResults = this.remoteFetcherService.getExternalGeneric(urlCriteria, genericUrls);
|
|
if (!singleResults.isEmpty() && !singleResults.get(0).containsKey("source") && !singleData.getAutoCompleteOptions().getSource().isEmpty()) {
|
|
singleResults.forEach(singleResult -> singleResult.put("source", singleData.getAutoCompleteOptions().getSource()));
|
|
}
|
|
rawResults.addAll(singleResults);
|
|
genericUrls.getUrls().clear();
|
|
} catch (URISyntaxException e) {
|
|
logger.error(e.getMessage(), e);
|
|
}
|
|
}
|
|
rawResults.forEach(item -> result.add(new ExternalAutocompleteFieldResult(parseItem(item.get("pid")), parseItem(item.get("name")), parseItem(item.get("source")), parseItem(item.get("uri")))));
|
|
return result;
|
|
}
|
|
|
|
private static String parseItem(Object item) {
|
|
if (item instanceof String) {
|
|
return (String) item;
|
|
}
|
|
if (item instanceof List) {
|
|
List listedItems = (List) item;
|
|
return parseItem(listedItems.get(0));
|
|
}
|
|
if (item instanceof Map) {
|
|
return String.valueOf(((Map)item).get("$"));
|
|
}
|
|
return item != null ? item.toString() : null;
|
|
}
|
|
|
|
public void addSemanticsInDatasetProfiles() throws XPathExpressionException, InvalidApplicationException {
|
|
List<DescriptionTemplateEntity> ids = this.databaseRepository.getDatasetProfileDao().getAllIds();
|
|
for(DescriptionTemplateEntity dp: ids){
|
|
DescriptionTemplateEntity descriptionTemplateEntity = this.databaseRepository.getDatasetProfileDao().find(dp.getId());
|
|
Document document = XmlBuilder.fromXml(descriptionTemplateEntity.getDefinition());
|
|
XPathFactory xpathFactory = XPathFactory.newInstance();
|
|
XPath xpath = xpathFactory.newXPath();
|
|
XPathExpression expr = xpath.compile("//rdaCommonStandard");
|
|
NodeList rdaProperties = (NodeList) expr.evaluate(document, XPathConstants.NODESET);
|
|
for(int i = 0; i < rdaProperties.getLength(); i++){
|
|
Node rdaPropertyNode = rdaProperties.item(i);
|
|
String rdaProperty = rdaPropertyNode.getTextContent();
|
|
Element schematics = document.createElement("schematics");
|
|
Node fieldParent = rdaPropertyNode.getParentNode();
|
|
if(rdaProperty != null && !rdaProperty.isEmpty()){
|
|
Element schematic = document.createElement("schematic");
|
|
schematic.setTextContent("rda." + rdaProperty);
|
|
schematics.appendChild(schematic);
|
|
}
|
|
fieldParent.insertBefore(schematics, rdaPropertyNode);
|
|
fieldParent.removeChild(rdaPropertyNode);
|
|
}
|
|
this.updateDatasetProfileXml(document, descriptionTemplateEntity);
|
|
}
|
|
}
|
|
|
|
public void addRdaInSemanticsInDatasetProfiles() throws XPathExpressionException, InvalidApplicationException {
|
|
List<DescriptionTemplateEntity> ids = this.databaseRepository.getDatasetProfileDao().getAllIds();
|
|
for(DescriptionTemplateEntity dp: ids){
|
|
DescriptionTemplateEntity descriptionTemplateEntity = this.databaseRepository.getDatasetProfileDao().find(dp.getId());
|
|
Document document = XmlBuilder.fromXml(descriptionTemplateEntity.getDefinition());
|
|
XPathFactory xpathFactory = XPathFactory.newInstance();
|
|
XPath xpath = xpathFactory.newXPath();
|
|
XPathExpression expr = xpath.compile("//schematic");
|
|
NodeList schematics = (NodeList) expr.evaluate(document, XPathConstants.NODESET);
|
|
for (int i = 0; i < schematics.getLength(); i++) {
|
|
Node schematicNode = schematics.item(i);
|
|
String schematicRda = schematicNode.getTextContent();
|
|
if (schematicRda != null && !schematicRda.isEmpty() && !schematicRda.startsWith("rda.")) {
|
|
schematicNode.setTextContent("rda." + schematicRda);
|
|
}
|
|
}
|
|
this.updateDatasetProfileXml(document, descriptionTemplateEntity);
|
|
}
|
|
}
|
|
|
|
private void updateDatasetProfileXml(Document document, DescriptionTemplateEntity descriptionTemplateEntity) {
|
|
try {
|
|
DOMSource domSource = new DOMSource(document);
|
|
StringWriter writer = new StringWriter();
|
|
StreamResult result = new StreamResult(writer);
|
|
TransformerFactory tf = TransformerFactory.newInstance();
|
|
Transformer transformer = tf.newTransformer();
|
|
transformer.setOutputProperty(OutputKeys.INDENT, "yes");
|
|
transformer.transform(domSource, result);
|
|
String newDefinition = writer.toString();
|
|
if(newDefinition != null){
|
|
descriptionTemplateEntity.setDefinition(newDefinition);
|
|
this.databaseRepository.getDatasetProfileDao().createOrUpdate(descriptionTemplateEntity);
|
|
}
|
|
}
|
|
catch(TransformerException ex) {
|
|
logger.error(ex.getMessage(), ex);
|
|
}
|
|
}
|
|
}
|