Re-configure RDA Mappers

This commit is contained in:
George Kalampokis 2020-05-28 18:38:08 +03:00
parent 20d0d8b648
commit d0952d91c5
4 changed files with 283 additions and 39 deletions

View File

@ -1,17 +1,86 @@
package eu.eudat.models.rda.mapper; package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.rda.DatasetId; import eu.eudat.models.rda.DatasetId;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID; import java.util.UUID;
public class DatasetIdRDAMapper { public class DatasetIdRDAMapper {
public static DatasetId toRDA(UUID id) { /*public static DatasetId toRDA(UUID id) {
DatasetId rda = new DatasetId(); DatasetId rda = new DatasetId();
rda.setIdentifier(id.toString()); rda.setIdentifier(id.toString());
rda.setType(DatasetId.Type.OTHER); rda.setType(DatasetId.Type.OTHER);
return rda; return rda;
}*/
public static DatasetId toRDA(List<JsonNode> nodes) {
DatasetId data = new DatasetId();
for (JsonNode node: nodes) {
String rdaProperty = node.get("rdaProperty").asText();
String rdaValue = node.get("value").asText();
for (DatasetIdProperties datasetIdProperties : DatasetIdProperties.values()) {
if (rdaProperty.contains(datasetIdProperties.getName())) {
switch (datasetIdProperties) {
case IDENTIFIER:
data.setIdentifier(rdaValue);
break;
case TYPE:
data.setType(DatasetId.Type.fromValue(rdaValue));
break;
}
}
}
}
if (data.getIdentifier() != null && data.getType() != null) {
return data;
}
return null;
}
public static Map<String, String> toProperties(DatasetId rda, JsonNode node) {
Map<String, String> properties = new HashMap<>();
List<JsonNode> idNodes = JsonSearcher.findNodes(node, "rdaProperty", "dataset.dataset_id");
for (JsonNode idNode: idNodes) {
for (DatasetIdProperties datasetIdProperties : DatasetIdProperties.values()) {
if (idNode.get("rdaProperty").asText().endsWith(datasetIdProperties.getName())) {
switch (datasetIdProperties) {
case IDENTIFIER:
properties.put(idNode.get("id").asText(), rda.getIdentifier());
break;
case TYPE:
properties.put(idNode.get("id").asText(), rda.getType().value());
break;
}
}
}
}
return properties;
}
private enum DatasetIdProperties {
IDENTIFIER("identifier"),
TYPE("type");
private final String name;
DatasetIdProperties(String name) {
this.name = name;
}
public String getName() {
return name;
}
} }
} }

View File

@ -3,11 +3,13 @@ package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.entities.DatasetProfile; import eu.eudat.data.entities.DatasetProfile;
import eu.eudat.elastic.entities.Tag;
import eu.eudat.logic.managers.DatasetManager; import eu.eudat.logic.managers.DatasetManager;
import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel; import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.rda.Dataset; import eu.eudat.models.rda.Dataset;
import eu.eudat.models.rda.DatasetId;
import org.json.JSONObject; import org.json.JSONObject;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
@ -35,7 +37,7 @@ public class DatasetRDAMapper {
@Transactional @Transactional
public Dataset toRDA(eu.eudat.data.entities.Dataset dataset) { public Dataset toRDA(eu.eudat.data.entities.Dataset dataset) {
Dataset rda = new Dataset(); Dataset rda = new Dataset();
rda.setDatasetId(DatasetIdRDAMapper.toRDA(dataset.getId())); // rda.setDatasetId(DatasetIdRDAMapper.toRDA(dataset.getId()));
rda.setTitle(dataset.getLabel()); rda.setTitle(dataset.getLabel());
rda.setDescription(dataset.getDescription()); rda.setDescription(dataset.getDescription());
rda.setAdditionalProperty("template", dataset.getProfile().getId()); rda.setAdditionalProperty("template", dataset.getProfile().getId());
@ -47,9 +49,15 @@ public class DatasetRDAMapper {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();
String datasetDescriptionJson = mapper.writeValueAsString(datasetWizardModel.getDatasetProfileDefinition()); String datasetDescriptionJson = mapper.writeValueAsString(datasetWizardModel.getDatasetProfileDefinition());
JsonNode datasetDescriptionObj = mapper.readTree(datasetDescriptionJson); JsonNode datasetDescriptionObj = mapper.readTree(datasetDescriptionJson);
List<JsonNode> idNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.dataset_id");
if (!idNodes.isEmpty()) {
rda.setDatasetId(DatasetIdRDAMapper.toRDA(idNodes));
}
List<JsonNode> typeNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.type"); List<JsonNode> typeNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.type");
if (!typeNodes.isEmpty()) { if (!typeNodes.isEmpty()) {
rda.setType(typeNodes.get(0).get("value").asText()); rda.setType(typeNodes.get(0).get("value").asText());
} else {
rda.setType(dataset.getLabel());
} }
List<JsonNode> languageNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.language"); List<JsonNode> languageNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.language");
if (!languageNodes.isEmpty()) { if (!languageNodes.isEmpty()) {
@ -61,10 +69,11 @@ public class DatasetRDAMapper {
} }
List<JsonNode> qaNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.data_quality_assurance"); List<JsonNode> qaNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.data_quality_assurance");
if (!qaNodes.isEmpty()) { if (!qaNodes.isEmpty()) {
rda.setDataQualityAssurance(qaNodes.stream().map(qaNode -> qaNode.get("value").asText()).collect(Collectors.toList())); /*rda.setDataQualityAssurance(qaNodes.stream().map(qaNode -> qaNode.get("value").asText()).collect(Collectors.toList()));
for (int i = 0; i < qaNodes.size(); i++) { for (int i = 0; i < qaNodes.size(); i++) {
rda.setAdditionalProperty("qaId" + (i + 1), qaNodes.get(i).get("id").asText()); rda.setAdditionalProperty("qaId" + (i + 1), qaNodes.get(i).get("id").asText());
} }*/
rda.setDataQualityAssurance(Collections.singletonList(qaNodes.get(0).get("value").asText()));
} }
List<JsonNode> preservationNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.preservation_statement"); List<JsonNode> preservationNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.preservation_statement");
if (!preservationNodes.isEmpty()) { if (!preservationNodes.isEmpty()) {
@ -72,7 +81,7 @@ public class DatasetRDAMapper {
} }
List<JsonNode> distributionNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.distribution"); List<JsonNode> distributionNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.distribution");
if (!distributionNodes.isEmpty()) { if (!distributionNodes.isEmpty()) {
rda.setDistribution(DistributionRDAMapper.toRDAList(distributionNodes)); rda.setDistribution(Collections.singletonList(DistributionRDAMapper.toRDA(distributionNodes)));
} }
List<JsonNode> keywordNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.keyword"); List<JsonNode> keywordNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.keyword");
if (!keywordNodes.isEmpty()) { if (!keywordNodes.isEmpty()) {
@ -80,10 +89,15 @@ public class DatasetRDAMapper {
for (int i = 0; i < keywordNodes.size(); i++) { for (int i = 0; i < keywordNodes.size(); i++) {
rda.setAdditionalProperty("keyword" + (i + 1), keywordNodes.get(i).get("id").asText()); rda.setAdditionalProperty("keyword" + (i + 1), keywordNodes.get(i).get("id").asText());
} }
} else {
List<String> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags().stream().map(Tag::getName).collect(Collectors.toList());
rda.setKeyword(tags);
} }
List<JsonNode> personalDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.personal_data"); List<JsonNode> personalDataNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.personal_data");
if (!personalDataNodes.isEmpty()) { if (!personalDataNodes.isEmpty()) {
rda.setPersonalData(personalDataNodes.stream().map(personalDataNode -> Dataset.PersonalData.fromValue(personalDataNode.get("value").asText())).findFirst().get()); rda.setPersonalData(personalDataNodes.stream().map(personalDataNode -> Dataset.PersonalData.fromValue(personalDataNode.get("value").asText())).findFirst().get());
} else {
rda.setPersonalData(Dataset.PersonalData.UNKNOWN);
} }
List<JsonNode> securityAndPrivacyNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.security_and_privacy"); List<JsonNode> securityAndPrivacyNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.security_and_privacy");
if (!securityAndPrivacyNodes.isEmpty()) { if (!securityAndPrivacyNodes.isEmpty()) {
@ -97,6 +111,10 @@ public class DatasetRDAMapper {
if (!technicalResourceNodes.isEmpty()) { if (!technicalResourceNodes.isEmpty()) {
rda.setTechnicalResource(TechnicalResourceRDAMapper.toRDAList(technicalResourceNodes)); rda.setTechnicalResource(TechnicalResourceRDAMapper.toRDAList(technicalResourceNodes));
} }
List<JsonNode> issuedNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.issued");
if (!issuedNodes.isEmpty()) {
rda.setIssued(issuedNodes.get(0).get("value").asText());
}
List<JsonNode> foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes, preservationNodes, distributionNodes, List<JsonNode> foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes, preservationNodes, distributionNodes,
keywordNodes, personalDataNodes, securityAndPrivacyNodes, sensitiveDataNodes, technicalResourceNodes).flatMap(Collection::stream).collect(Collectors.toList()); keywordNodes, personalDataNodes, securityAndPrivacyNodes, sensitiveDataNodes, technicalResourceNodes).flatMap(Collection::stream).collect(Collectors.toList());
templateIdsToValues.entrySet().forEach(entry -> { templateIdsToValues.entrySet().forEach(entry -> {
@ -149,9 +167,17 @@ public class DatasetRDAMapper {
properties.putAll(MetadataRDAMapper.toProperties(rda.getMetadata())); properties.putAll(MetadataRDAMapper.toProperties(rda.getMetadata()));
} }
List <String> qaIds = rda.getAdditionalProperties().entrySet().stream().filter(entry -> entry.getKey().startsWith("qaId")).map(entry -> entry.getValue().toString()).collect(Collectors.toList()); if (rda.getDatasetId() != null) {
properties.putAll(DatasetIdRDAMapper.toProperties(rda.getDatasetId(), datasetDescriptionObj));
}
/*List <String> qaIds = rda.getAdditionalProperties().entrySet().stream().filter(entry -> entry.getKey().startsWith("qaId")).map(entry -> entry.getValue().toString()).collect(Collectors.toList());
for (int i = 0; i < qaIds.size(); i++) { for (int i = 0; i < qaIds.size(); i++) {
properties.put(qaIds.get(i), rda.getDataQualityAssurance().get(i)); properties.put(qaIds.get(i), rda.getDataQualityAssurance().get(i));
}*/
List<JsonNode> qaNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.data_quality_assurance");
if (!qaNodes.isEmpty()) {
properties.put(qaNodes.get(0).get("id").asText(), rda.getDataQualityAssurance().get(0));
} }
List<JsonNode> preservationNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.preservation_statement"); List<JsonNode> preservationNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.preservation_statement");
@ -159,8 +185,13 @@ public class DatasetRDAMapper {
properties.put(preservationNodes.get(0).get("id").asText(), rda.getPreservationStatement()); properties.put(preservationNodes.get(0).get("id").asText(), rda.getPreservationStatement());
} }
List<JsonNode> issuedNodes = JsonSearcher.findNodes(datasetDescriptionObj, "rdaProperty", "dataset.issued");
if (!issuedNodes.isEmpty()) {
properties.put(issuedNodes.get(0).get("id").asText(), rda.getIssued());
}
if (rda.getDistribution() != null) { if (rda.getDistribution() != null) {
properties.putAll(DistributionRDAMapper.toProperties(rda.getDistribution())); properties.putAll(DistributionRDAMapper.toProperties(rda.getDistribution().get(0), datasetDescriptionObj));
} }
List <String> keywordIds = rda.getAdditionalProperties().entrySet().stream().filter(entry -> entry.getKey().startsWith("keyword")).map(entry -> entry.getValue().toString()).collect(Collectors.toList()); List <String> keywordIds = rda.getAdditionalProperties().entrySet().stream().filter(entry -> entry.getKey().startsWith("keyword")).map(entry -> entry.getValue().toString()).collect(Collectors.toList());

View File

@ -2,12 +2,14 @@ package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import eu.eudat.logic.utilities.helpers.MyStringUtils; import eu.eudat.logic.utilities.helpers.MyStringUtils;
import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.rda.Distribution; import eu.eudat.models.rda.Distribution;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import java.net.URI; import java.net.URI;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
public class DistributionRDAMapper { public class DistributionRDAMapper {
private static final Logger logger = LoggerFactory.getLogger(DistributionRDAMapper.class); private static final Logger logger = LoggerFactory.getLogger(DistributionRDAMapper.class);
@ -50,7 +52,8 @@ public class DistributionRDAMapper {
rda.setAdditionalProperty(ImportPropertyName.BYTE_SIZE.getName(), node.get("id").asText()); rda.setAdditionalProperty(ImportPropertyName.BYTE_SIZE.getName(), node.get("id").asText());
break; break;
case LICENSE: case LICENSE:
rda.setLicense(Collections.singletonList(LicenseRDAMapper.toRDA(node))); List<JsonNode> licenseNodes = nodes.stream().filter(lnode -> lnode.get("rdaProperty").asText().toLowerCase().contains("license")).collect(Collectors.toList());
rda.setLicense(Collections.singletonList(LicenseRDAMapper.toRDA(licenseNodes)));
break; break;
case FORMAT: case FORMAT:
rda.setFormat(Collections.singletonList(rdaValue)); rda.setFormat(Collections.singletonList(rdaValue));
@ -120,11 +123,96 @@ public class DistributionRDAMapper {
return properties; return properties;
} }
public static Distribution toRDA(JsonNode node) { public static Map<String, String> toProperties(Distribution rda, JsonNode root) {
Map<String, String> properties = new HashMap<>();
List<JsonNode> distributionNodes = JsonSearcher.findNodes(root, "rdaProperty", "dataset.distribution");
for (JsonNode distributionNode: distributionNodes) {
for (ExportPropertyName exportPropertyName: ExportPropertyName.values()) {
if (distributionNode.get("rdaProperty").asText().endsWith(exportPropertyName.getName())) {
switch (exportPropertyName) {
case ACCESS_URL:
properties.put(distributionNode.get("id").asText(), rda.getAccessUrl());
break;
case DESCRIPTION:
properties.put(distributionNode.get("id").asText(), rda.getDescription());
break;
case TITLE:
properties.put(distributionNode.get("id").asText(), rda.getTitle());
break;
case AVAILABLE_UTIL:
properties.put(distributionNode.get("id").asText(), rda.getAvailableUntil());
break;
case DOWNLOAD_URL:
properties.put(distributionNode.get("id").asText(), rda.getDownloadUrl().toString());
break;
case DATA_ACCESS:
properties.put(distributionNode.get("id").asText(), rda.getDataAccess().value());
break;
case BYTE_SIZE:
properties.put(distributionNode.get("id").asText(), rda.getByteSize().toString());
break;
case FORMAT:
properties.put(distributionNode.get("id").asText(), rda.getFormat().get(0));
break;
case LICENSE:
properties.putAll(LicenseRDAMapper.toProperties(rda.getLicense().get(0), root));
break;
case HOST:
properties.putAll(HostRDAMapper.toProperties(rda.getHost()));
}
}
}
}
return properties;
}
public static Distribution toRDA(List<JsonNode> nodes) {
Distribution rda = new Distribution(); Distribution rda = new Distribution();
for (JsonNode node: nodes) {
String rdaProperty = node.get("rdaProperty").asText(); String rdaProperty = node.get("rdaProperty").asText();
String rdaValue = node.get("value").asText(); String rdaValue = node.get("value").asText();
if (rdaProperty.contains("access_url")) { for (ExportPropertyName exportPropertyName: ExportPropertyName.values()) {
if (rdaProperty.contains(exportPropertyName.getName())) {
switch (exportPropertyName) {
case ACCESS_URL:
rda.setAccessUrl(rdaValue);
break;
case DESCRIPTION:
rda.setDescription(rdaValue);
break;
case TITLE:
rda.setTitle(rdaValue);
break;
case AVAILABLE_UTIL:
rda.setAvailableUntil(rdaValue);
break;
case DOWNLOAD_URL:
rda.setDownloadUrl(URI.create(rdaValue));
break;
case DATA_ACCESS:
rda.setDataAccess(Distribution.DataAccess.fromValue(rdaValue));
break;
case BYTE_SIZE:
rda.setByteSize(Integer.parseInt(rdaValue));
break;
case FORMAT:
rda.setFormat(Collections.singletonList(rdaValue));
break;
case LICENSE:
List<JsonNode> licenseNodes = nodes.stream().filter(lnode -> lnode.get("rdaProperty").asText().toLowerCase().contains("license")).collect(Collectors.toList());
rda.setLicense(Collections.singletonList(LicenseRDAMapper.toRDA(licenseNodes)));
break;
case HOST:
List<JsonNode> hostNodes = nodes.stream().filter(lnode -> lnode.get("rdaProperty").asText().toLowerCase().contains("host")).collect(Collectors.toList());
rda.setHost(HostRDAMapper.toRDA(hostNodes, "0"));
break;
}
}
}
/*if (rdaProperty.contains("access_url")) {
rda.setAccessUrl(rdaValue); rda.setAccessUrl(rdaValue);
} else if (rdaProperty.contains("available_util")) { } else if (rdaProperty.contains("available_util")) {
rda.setAvailableUntil(rdaValue); rda.setAvailableUntil(rdaValue);
@ -144,6 +232,8 @@ public class DistributionRDAMapper {
rda.setLicense(Collections.singletonList(LicenseRDAMapper.toRDA(node))); rda.setLicense(Collections.singletonList(LicenseRDAMapper.toRDA(node)));
} else if (rdaProperty.contains("title")) { } else if (rdaProperty.contains("title")) {
rda.setTitle(rdaValue); rda.setTitle(rdaValue);
}*/
} }

View File

@ -1,6 +1,7 @@
package eu.eudat.models.rda.mapper; package eu.eudat.models.rda.mapper;
import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.JsonNode;
import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.rda.License; import eu.eudat.models.rda.License;
import java.net.URI; import java.net.URI;
@ -10,17 +11,32 @@ import java.util.Map;
public class LicenseRDAMapper { public class LicenseRDAMapper {
public static License toRDA(JsonNode node) { public static License toRDA(List<JsonNode> nodes) {
License rda = new License(); License rda = new License();
for (JsonNode node: nodes) {
String rdaProperty = node.get("rdaProperty").asText(); String rdaProperty = node.get("rdaProperty").asText();
String value = node.get("value").asText(); String value = node.get("value").asText();
if (rdaProperty.contains("license_ref")) { for (LicenceProperties licenceProperties: LicenceProperties.values()) {
if (rdaProperty.contains(licenceProperties.getName())) {
switch (licenceProperties) {
case LICENSE_REF:
rda.setLicenseRef(URI.create(value));
break;
case START_DATE:
rda.setStartDate(value);
break;
}
}
}
/*if (rdaProperty.contains("license_ref")) {
rda.setLicenseRef(URI.create(value)); rda.setLicenseRef(URI.create(value));
rda.setAdditionalProperty("license_refId", node.get("id").asText()); rda.setAdditionalProperty("license_refId", node.get("id").asText());
} else if (rdaProperty.contains("start_date")) { } else if (rdaProperty.contains("start_date")) {
rda.setStartDate(value); rda.setStartDate(value);
rda.setAdditionalProperty("start_dateId", node.get("id").asText()); rda.setAdditionalProperty("start_dateId", node.get("id").asText());
}*/
} }
return rda; return rda;
@ -45,4 +61,42 @@ public class LicenseRDAMapper {
return properties; return properties;
} }
public static Map<String, String> toProperties(License rda, JsonNode root) {
Map<String, String> properties = new HashMap<>();
List<JsonNode> licenseNodes = JsonSearcher.findNodes(root, "rdaProperty", "dataset.distribution.license");
for (JsonNode licenseNode: licenseNodes) {
for (LicenceProperties licenceProperty: LicenceProperties.values()) {
if (licenseNode.get("rdaProperty").asText().endsWith(licenceProperty.getName())) {
switch (licenceProperty) {
case LICENSE_REF:
properties.put(licenseNode.get("id").asText(), rda.getLicenseRef().toString());
break;
case START_DATE:
properties.put(licenseNode.get("id").asText(), rda.getStartDate());
break;
}
}
}
}
return properties;
}
public enum LicenceProperties {
LICENSE_REF("license_ref"),
START_DATE("start_date");
private String name;
LicenceProperties(String name) {
this.name = name;
}
public String getName() {
return name;
}
}
} }