MaDgiK-CITE
/
file-transformer-rda-json
13
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

update commons

This commit is contained in:
Efstratios Giannopoulos 2024-05-03 13:31:15 +03:00
parent 5242120aea
commit 729d3f9d56
11 changed files with 497 additions and 428 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
core/src/main/java/org/opencdmp/filetransformer/rda/model/rda/mapper/CostRDAMapper.java
View File

@ -15,53 +15,48 @@ import java.util.stream.Collectors;
public class CostRDAMapper{
private static final Logger logger = LoggerFactory.getLogger(DatasetRDAMapper.class);
public List<Cost> toRDA(List<FieldModel> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields ) {
public List<Cost> toRDA(Map<String, List<FieldModel>> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields ) {
if (nodes == null) return null;
if (valueFields == null) throw new IllegalArgumentException("valueFields is missing");
Map<String, Cost> rdaMap = new HashMap<>();
for(FieldModel node: nodes){
for(Map.Entry<String, List<FieldModel>> entry: nodes.entrySet()){
for(FieldModel node: entry.getValue()) {
String rdaProperty = node.getSchematics().stream().filter(schematic -> schematic.startsWith("rda.dmp.cost")).findFirst().orElse("");
org.opencdmp.commonmodels.models.description.FieldModel rdaValue = valueFields.stream().filter(x -> x.getId().equals(node.getId())).findFirst().orElse(null);
if (rdaValue == null || rdaValue.getTextValue() == null || rdaValue.getTextValue().isBlank()) {
continue;
}
String key = node.getNumbering();
String key = entry.getKey();
if (!key.contains("mult")) {
key = "0";
}
else{
} else {
key = "" + key.charAt(4);
}
Cost rda;
if (rdaMap.containsKey(key)) {
rda = rdaMap.get(key);
}
else{
} else {
rda = new Cost();
rdaMap.put(key, rda);
}
if (rdaProperty.contains("value")) {
try {
rda.setValue(Double.valueOf(rdaValue.getTextValue()));
}
catch (NumberFormatException e) {
} catch (NumberFormatException e) {
logger.warn("Dmp cost value " + rdaValue + " is not valid. Cost value will not be set.");
}
}
else if(rdaProperty.contains("currency_code")){
} else if (rdaProperty.contains("currency_code")) {
try {
// HashMap<String, String> result =
// new ObjectMapper().readValue(rdaValue, HashMap.class);
// rda.setCurrencyCode(Cost.CurrencyCode.fromValue(result.get("value")));
rda.setCurrencyCode(Cost.CurrencyCode.fromValue(rdaValue.getTextValue())); //TODO
}
catch (Exception e) {
} catch (Exception e) {
logger.warn("Dmp cost currency code is not valid and will not be set.");
}
}
else if(rdaProperty.contains("title")){
} else if (rdaProperty.contains("title")) {
rda.setTitle(rdaValue.getTextValue());
// Iterator<JsonNode> iter = mapper.readTree(rdaValue).elements(); //TODO
@ -84,11 +79,11 @@ public class CostRDAMapper{
// rda.setTitle(t + ", " + rdaValue);
// }
// }
}
else if(rdaProperty.contains("description")){
} else if (rdaProperty.contains("description")) {
rda.setDescription(rdaValue.getTextValue());
}
}
}
List<Cost> rdaList = rdaMap.values().stream()
.filter(cost -> cost.getTitle() != null)
.collect(Collectors.toList());

7
core/src/main/java/org/opencdmp/filetransformer/rda/model/rda/mapper/DatasetIdRDAMapper.java
View File

@ -14,12 +14,13 @@ public class DatasetIdRDAMapper{
private static final Logger logger = LoggerFactory.getLogger(DatasetIdRDAMapper.class);
public DatasetId toRDA(List<FieldModel> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields) {
public DatasetId toRDA(Map<String, List<FieldModel>> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields) {
if (nodes == null) return null;
if (valueFields == null) throw new IllegalArgumentException("valueFields is missing");
DatasetId data = new DatasetId();
for (FieldModel node: nodes) {
for (Map.Entry<String, List<FieldModel>> entry: nodes.entrySet()) {
for (FieldModel node: entry.getValue()) {
String rdaProperty = node.getSchematics().stream().filter(schematic -> schematic.startsWith("rda.dataset.dataset_id")).findFirst().orElse("");
if (node.getData() == null) {
continue;
@ -41,7 +42,7 @@ public class DatasetIdRDAMapper{
// finalRDAMap(data, rdaProperty, rdaValue.getTextValue());
//
// }
}
}
if (data.getIdentifier() != null && data.getType() != null) {

80
core/src/main/java/org/opencdmp/filetransformer/rda/model/rda/mapper/DatasetRDAMapper.java
View File

@ -92,7 +92,7 @@ public class DatasetRDAMapper {
//Map<String, Object> templateIdsToValues = this.createFieldIdValueMap(descriptionEntity.getDescriptionTemplate());
//rda.setAdditionalProperty("template", descriptionEntity.getDescriptionTemplate()); //TODO
try {
List<FieldModel> idNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.dataset_id");
Map<String, List<FieldModel>> idNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.dataset_id");
if (!idNodes.isEmpty()) {
rda.setDatasetId(datasetIdRDAMapper.toRDA(idNodes, this.getAllValueFields(descriptionEntity.getProperties())));
}
@ -100,31 +100,34 @@ public class DatasetRDAMapper {
rda.setDatasetId(new DatasetId(descriptionEntity.getId().toString(), DatasetId.Type.OTHER));
}
List<FieldModel> typeNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.type");
for (FieldModel typeNode : typeNodes){
Map<String, List<FieldModel>> typeNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.type");
for (Map.Entry<String, List<FieldModel>> entry : typeNodes.entrySet()){
for (FieldModel typeNode : entry.getValue()) {
List<org.opencdmp.commonmodels.models.description.FieldModel> fieldValues = this.findValueField(typeNode, descriptionEntity.getProperties());
if (!fieldValues.isEmpty()) rda.setType(fieldValues.stream().filter(x -> x.getTextValue() != null && x.getTextValue().isBlank()).map(org.opencdmp.commonmodels.models.description.FieldModel::getTextValue).findFirst().orElse(null));
if (rda.getType() != null && !rda.getType().isBlank()) break;;
if (rda.getType() != null && !rda.getType().isBlank()) break;
}
}
if (rda.getType() == null || rda.getType().isBlank()) rda.setType("DMP Dataset");
List<FieldModel> languageNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.language");
for (FieldModel languageNode : languageNodes){
Map<String, List<FieldModel>> languageNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.language");
for (Map.Entry<String, List<FieldModel>> entry : languageNodes.entrySet()){
for (FieldModel languageNode : entry.getValue()) {
List<org.opencdmp.commonmodels.models.description.FieldModel> fieldValues = this.findValueField(languageNode, descriptionEntity.getProperties());
try {
if (!fieldValues.isEmpty()) rda.setLanguage(fieldValues.stream().filter(x -> x.getTextValue() != null && x.getTextValue().isBlank()).map(x -> Language.fromValue(x.getTextValue())).findFirst().orElse(null));
}
catch (IllegalArgumentException e){
} catch (IllegalArgumentException e) {
logger.warn("Language from semantic rda.dataset.language was not found.");
}
if (rda.getLanguage() != null) break;;
if (rda.getLanguage() != null) break;
}
}
if (rda.getLanguage() == null) rda.setLanguage(languageRDAMapper.toRDA(descriptionEntity.getDescriptionTemplate().getLanguage()));
List<FieldModel> metadataNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.metadata");
Map<String, List<FieldModel>> metadataNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.metadata");
if (!metadataNodes.isEmpty()) {
Map<String, Object> valueFieldsMap = new HashMap<>();
valueFieldsMap.put("valueFields", this.getAllValueFields(descriptionEntity.getProperties()));
@ -173,15 +176,17 @@ public class DatasetRDAMapper {
// }
List<FieldModel> preservationNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.preservation_statement");
for (FieldModel preservationNode : preservationNodes){
Map<String, List<FieldModel>> preservationNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.preservation_statement");
for (Map.Entry<String, List<FieldModel>> entry : preservationNodes.entrySet()){
for (FieldModel preservationNode : entry.getValue()){
List<org.opencdmp.commonmodels.models.description.FieldModel> fieldValues = this.findValueField(preservationNode, descriptionEntity.getProperties());
if (!fieldValues.isEmpty()) rda.setPreservationStatement(fieldValues.stream().filter(x-> x.getTextValue() != null && x.getTextValue().isBlank()).map(org.opencdmp.commonmodels.models.description.FieldModel::getTextValue).findFirst().orElse(null));
if (rda.getPreservationStatement() != null && !rda.getPreservationStatement().isBlank()) break;;
if (rda.getPreservationStatement() != null && !rda.getPreservationStatement().isBlank()) break;
}
}
List<FieldModel> distributionNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.distribution");
Map<String, List<FieldModel>> distributionNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.distribution");
if (!distributionNodes.isEmpty()) {
rda.setDistribution(distributionRDAMapper.toRDA(distributionNodes, this.getAllValueFields(descriptionEntity.getProperties())));
}else{
@ -189,8 +194,9 @@ public class DatasetRDAMapper {
}
List<FieldModel> keywordNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.keyword");
for (FieldModel keywordNode : keywordNodes){
Map<String, List<FieldModel>> keywordNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.keyword");
for (Map.Entry<String, List<FieldModel>> entry : keywordNodes.entrySet()) {
for (FieldModel keywordNode : entry.getValue()) {
List<org.opencdmp.commonmodels.models.description.FieldModel> fieldValues = this.findValueField(keywordNode, descriptionEntity.getProperties());
if (!fieldValues.isEmpty()) {
rda.setKeyword(fieldValues.stream().filter(x -> (x.getTextValue() != null && x.getTextValue().isBlank()) || (x.getTextListValue() != null && !x.getTextListValue().isEmpty())).map(x -> {
@ -202,6 +208,7 @@ public class DatasetRDAMapper {
}).filter(Objects::nonNull).flatMap(Collection::stream).collect(Collectors.toList()));
}
}
}
if (rda.getKeyword() != null){
int i = 0 ;
for (String keyword : rda.getKeyword()) {
@ -214,40 +221,46 @@ public class DatasetRDAMapper {
// List<String> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(descriptionEntity.getId().toString()).getTags().stream().map(Tag::getName).collect(Collectors.toList());
// rda.setKeyword(tags);
// }
List<FieldModel> personalDataNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.sensitive_data");
for (FieldModel personalDataNode : personalDataNodes){
Map<String, List<FieldModel>> personalDataNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.sensitive_data");
for (Map.Entry<String, List<FieldModel>> entry : personalDataNodes.entrySet()) {
for (FieldModel personalDataNode : entry.getValue()) {
List<org.opencdmp.commonmodels.models.description.FieldModel> fieldValues = this.findValueField(personalDataNode, descriptionEntity.getProperties());
if (!fieldValues.isEmpty()) rda.setPersonalData(fieldValues.stream().filter(x -> x.getTextValue() != null && x.getTextValue().isBlank()).map(x -> Dataset.PersonalData.fromValue(x.getTextValue())).findFirst().orElse(null));
if (rda.getPersonalData() != null) break;
}
}
if (rda.getPersonalData() != null) rda.setPersonalData(Dataset.PersonalData.UNKNOWN);
List<FieldModel> securityAndPrivacyNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.security_and_privacy");
Map<String, List<FieldModel>> securityAndPrivacyNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.security_and_privacy");
if (!securityAndPrivacyNodes.isEmpty()) {
rda.setSecurityAndPrivacy(securityAndPrivacyRDAMapper.toRDA(securityAndPrivacyNodes, this.getAllValueFields(descriptionEntity.getProperties())));
}else{
rda.setSecurityAndPrivacy(new ArrayList<>());
}
List<FieldModel> sensitiveDataNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.sensitive_data");
for (FieldModel sensitiveDataNode : sensitiveDataNodes){
Map<String, List<FieldModel>> sensitiveDataNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.sensitive_data");
for (Map.Entry<String, List<FieldModel>> entry : sensitiveDataNodes.entrySet()) {
for (FieldModel sensitiveDataNode : entry.getValue()) {
List<org.opencdmp.commonmodels.models.description.FieldModel> fieldValues = this.findValueField(sensitiveDataNode, descriptionEntity.getProperties());
if (!fieldValues.isEmpty()) rda.setSensitiveData(fieldValues.stream().filter(x -> x.getTextValue() != null && x.getTextValue().isBlank()).map(x -> Dataset.SensitiveData.fromValue(x.getTextValue())).findFirst().orElse(null));
if (rda.getSensitiveData() != null) break;
}
}
if (rda.getSensitiveData() != null) rda.setSensitiveData(Dataset.SensitiveData.UNKNOWN);
List<FieldModel> technicalResourceNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.technical_resource");
Map<String, List<FieldModel>> technicalResourceNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.technical_resource");
if (!technicalResourceNodes.isEmpty()) {
rda.setTechnicalResource(technicalResourceRDAMapper.toRDA(technicalResourceNodes, this.getAllValueFields(descriptionEntity.getProperties())));
}else{
rda.setTechnicalResource(new ArrayList<>());
}
List<FieldModel> issuedNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.issued");
for (FieldModel issuedNode : issuedNodes){
Map<String, List<FieldModel>> issuedNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dataset.issued");
for (Map.Entry<String, List<FieldModel>> entry : issuedNodes.entrySet()) {
for (FieldModel issuedNode : entry.getValue()) {
List<org.opencdmp.commonmodels.models.description.FieldModel> fieldValues = this.findValueField(issuedNode, descriptionEntity.getProperties());
if (!fieldValues.isEmpty()) rda.setIssued(fieldValues.stream().filter(x -> x.getTextValue() != null && x.getTextValue().isBlank()).map(org.opencdmp.commonmodels.models.description.FieldModel::getTextValue).findFirst().orElse(null));
if (rda.getIssued() != null && !rda.getIssued().isBlank()) break;;
if (rda.getIssued() != null && !rda.getIssued().isBlank()) break;
}
}
//TODO
@ -267,13 +280,14 @@ public class DatasetRDAMapper {
// }).filter(Objects::nonNull).flatMap(Collection::stream).toList());
// dmp.setContributor(dmp.getContributor().stream().filter(contributor -> contributor.getContributorId() != null && contributor.getName() != null).collect(Collectors.toList()));
// }
List<FieldModel> costNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dmp.cost");
Map<String, List<FieldModel>> costNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dmp.cost");
if (!costNodes.isEmpty()) {
dmp.getCost().addAll(costRDAMapper.toRDA(costNodes, this.getAllValueFields(descriptionEntity.getProperties())));
}
List<FieldModel> ethicsNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dmp.ethical_issues");
Map<String, List<FieldModel>> ethicsNodes = this.templateFieldSearcherService.searchFieldsBySemantics(descriptionEntity.getDescriptionTemplate(), "rda.dmp.ethical_issues");
if (!ethicsNodes.isEmpty()) {
for(FieldModel node: ethicsNodes){
for(Map.Entry<String, List<FieldModel>> entry: ethicsNodes.entrySet()) {
for (FieldModel node : entry.getValue()) {
List<org.opencdmp.commonmodels.models.description.FieldModel> fieldValues = this.findValueField(node, descriptionEntity.getProperties());
org.opencdmp.commonmodels.models.description.FieldModel fieldValue = fieldValues.getFirst();
String rdaProperty = node.getSchematics().stream().filter(schematic -> schematic.startsWith("rda.dmp.ethical_issues")).findFirst().orElse("");
@ -297,16 +311,13 @@ public class DatasetRDAMapper {
logger.warn(e.getLocalizedMessage() + ". Setting ethical_issues_exist to unknown");
dmp.setEthicalIssuesExist(Dmp.EthicalIssuesExist.UNKNOWN);
}
}
else if(rdaProperty.contains("description")){
} else if (rdaProperty.contains("description")) {
if (dmp.getEthicalIssuesDescription() == null) {
dmp.setEthicalIssuesDescription(rdaValue);
}
else{
} else {
dmp.setEthicalIssuesDescription(dmp.getEthicalIssuesDescription() + ", " + rdaValue);
}
}
else if(rdaProperty.contains("report")){
} else if (rdaProperty.contains("report")) {
try {
dmp.setEthicalIssuesReport(URI.create(rdaValue));
} catch (IllegalArgumentException e) {
@ -315,6 +326,7 @@ public class DatasetRDAMapper {
}
}
}
}
//TODO
// List<FieldModel> foundNodes = Stream.of(typeNodes, languageNodes, metadataNodes, qaNodes , preservationNodes, distributionNodes,
// keywordNodes, personalDataNodes, securityAndPrivacyNodes, sensitiveDataNodes, technicalResourceNodes).flatMap(Collection::stream).toList();

21
core/src/main/java/org/opencdmp/filetransformer/rda/model/rda/mapper/DistributionRDAMapper.java
View File

@ -22,12 +22,13 @@ public class DistributionRDAMapper {
this.hostRDAMapper = hostRDAMapper;
}
public List<Distribution> toRDA(List<FieldModel> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields) {
public List<Distribution> toRDA(Map<String, List<FieldModel>> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields) {
if (nodes == null) return null;
if (valueFields == null) throw new IllegalArgumentException("valueFields is missing");
Map<String, Distribution> rdaMap = new HashMap<>();
for (FieldModel node: nodes) {
for (Map.Entry<String, List<FieldModel>> entry: nodes.entrySet()) {
for (FieldModel node: entry.getValue()) {
String rdaProperty = getRdaDistributionProperty(node);
if (rdaProperty.isEmpty() || node.getData() == null) {
continue;
@ -37,18 +38,16 @@ public class DistributionRDAMapper {
if (rdaValue == null || (rdaValue.getTextValue() == null && rdaValue.getReferences() == null)) {
continue;
}
String key = node.getNumbering();
String key = entry.getKey();
if (!key.contains("mult")) {
key = "0";
}
else{
} else {
key = "" + key.charAt(4);
}
Distribution rda;
if (rdaMap.containsKey(key)) {
rda = rdaMap.get(key);
}
else {
} else {
rda = new Distribution();
rdaMap.put(key, rda);
}
@ -77,8 +76,7 @@ public class DistributionRDAMapper {
try {
rda.setDataAccess(Distribution.DataAccess.fromValue(rdaValue.getTextValue()));
rda.setAdditionalProperty(ImportPropertyName.DATA_ACCESS.getName(), node.getId());
}
catch (IllegalArgumentException e) {
} catch (IllegalArgumentException e) {
logger.warn("Distribution data access " + rdaValue + " from semantic distribution.data_access is not valid. Data access will not be set set.");
}
break;
@ -87,7 +85,7 @@ public class DistributionRDAMapper {
rda.setAdditionalProperty(ImportPropertyName.BYTE_SIZE.getName(), node.getId());
break;
case LICENSE:
List<FieldModel> licenseNodes = nodes.stream().filter(lnode -> {
List<FieldModel> licenseNodes = nodes.values().stream().flatMap(Collection::stream).filter(lnode -> {
for (String schematic : lnode.getSchematics()) {
if (schematic.startsWith("rda.dataset.distribution.license")) {
return true;
@ -139,12 +137,13 @@ public class DistributionRDAMapper {
}
break;
case HOST:
rda.setHost(hostRDAMapper.toRDA(nodes, valueFields, node.getNumbering()));
rda.setHost(hostRDAMapper.toRDA(nodes, valueFields, entry.getKey()));
break;
}
}
}
}
}
return rdaMap.values().stream()
.filter(distro -> distro.getTitle() != null).collect(Collectors.toList());
}

14
core/src/main/java/org/opencdmp/filetransformer/rda/model/rda/mapper/HostRDAMapper.java
View File

@ -14,15 +14,16 @@ import java.util.*;
public class HostRDAMapper {
private static final Logger logger = LoggerFactory.getLogger(HostRDAMapper.class);
public Host toRDA(List<FieldModel> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields, String numbering){
public Host toRDA(Map<String, List<FieldModel>> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields, String numbering){
if (nodes == null) return null;
if (valueFields == null) throw new IllegalArgumentException("valueFields is missing");
Host rda = new Host();
for (FieldModel node: nodes) {
for (Map.Entry<String, List<FieldModel>> entry: nodes.entrySet()) {
for (FieldModel node: entry.getValue()) {
String rdaProperty = node.getSchematics().stream().filter(schematic -> schematic.startsWith("rda.dataset.distribution.host")).findFirst().orElse("");
if (rdaProperty.contains("host")) {
int firstDiff = MyStringUtils.getFirstDifference(numbering, node.getNumbering());
int firstDiff = MyStringUtils.getFirstDifference(numbering, entry.getKey());
if (firstDiff == -1 || firstDiff >= 2) {
org.opencdmp.commonmodels.models.description.FieldModel rdaValue = valueFields.stream().filter(x -> x.getId().equals(node.getId())).findFirst().orElse(null);
if (rdaValue == null || rdaValue.getTextValue() == null || rdaValue.getTextValue().isBlank()) {
@ -47,8 +48,7 @@ public class HostRDAMapper {
try {
rda.setCertifiedWith(Host.CertifiedWith.fromValue(rdaValue.getTextValue()));
rda.setAdditionalProperty(ImportPropertyName.CERTIFIED_WITH.getName(), node.getId());
}
catch (IllegalArgumentException e) {
} catch (IllegalArgumentException e) {
logger.warn("Distribution host certified with " + rdaValue + "from semantic distribution.host.certified_with is not valid. Certified_with will not be set set.");
}
break;
@ -110,8 +110,7 @@ public class HostRDAMapper {
try {
rda.setSupportVersioning(Host.SupportVersioning.fromValue(rdaValue.getTextValue()));
rda.setAdditionalProperty(ImportPropertyName.SUPPORT_VERSIONING.getName(), node.getId());
}
catch (IllegalArgumentException e) {
} catch (IllegalArgumentException e) {
logger.warn("Distribution host support versioning " + rdaValue + "from semantic distribution.host.support_versioning is not valid. Support versioning will not be set set.");
}
break;
@ -133,6 +132,7 @@ public class HostRDAMapper {
}
}
}
}
if(rda.getTitle() == null || rda.getUrl() == null){
return null;

15
core/src/main/java/org/opencdmp/filetransformer/rda/model/rda/mapper/MetadataRDAMapper.java
View File

@ -22,13 +22,14 @@ public class MetadataRDAMapper{
this.metadataStandardIdRDAMapper = metadataStandardIdRDAMapper;
}
public List<Metadatum> toRDA(List<FieldModel> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields) {
public List<Metadatum> toRDA(Map<String, List<FieldModel>> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields) {
if (nodes == null) return null;
if (valueFields == null) throw new IllegalArgumentException("valueFields is missing");
Map<String, String> rdaMap = new HashMap<>();
List<Metadatum> rdas = new ArrayList<>();
for (FieldModel node : nodes) {
for (Map.Entry<String, List<FieldModel>> entry : nodes.entrySet()) {
for (FieldModel node : entry.getValue()) {
String rdaProperty = node.getSchematics().stream().filter(schematic -> schematic.startsWith("rda.dataset.metadata")).findFirst().orElse("");
org.opencdmp.commonmodels.models.description.FieldModel rdaValue = valueFields.stream().filter(x -> x.getId().equals(node.getId())).findFirst().orElse(null);
@ -45,19 +46,19 @@ public class MetadataRDAMapper{
rdas.getLast().setDescription(referenceModel.getLabel());
rdas.getLast().setAdditionalProperty("fieldId", node.getId());
rdas.getLast().setAdditionalProperty("valueId", referenceModel.getReference());
rdaMap.put(uri, node.getNumbering());
rdaMap.put(uri, entry.getKey());
}
}
} else if (rdaValue != null && rdaValue.getTextValue() != null && !rdaValue.getTextValue().isEmpty()) {
rdas.add(new Metadatum());
rdas.getLast().setMetadataStandardId(metadataStandardIdRDAMapper.toRDA(rdaValue.getTextValue()));
rdas.getLast().setAdditionalProperty("identifierId", node.getId());
rdaMap.put(rdaValue.getTextValue(), node.getNumbering());
rdaMap.put(rdaValue.getTextValue(), entry.getKey());
}
break;
case DESCRIPTION:
if (rdaValue != null && rdaValue.getTextValue() != null && !rdaValue.getTextValue().isEmpty()) {
Metadatum rda = getRelative(rdas, rdaMap, node.getNumbering());
Metadatum rda = getRelative(rdas, rdaMap, entry.getKey());
if (rda != null) {
rda.setDescription(rdaValue.getTextValue());
rda.setAdditionalProperty("descriptionId", node.getId());
@ -70,7 +71,7 @@ public class MetadataRDAMapper{
if (rdaValue != null && rdaValue.getTextValue() != null && !rdaValue.getTextValue().isEmpty()) {
String language = rdaValue.getTextValue();
Metadatum.Language lang = Metadatum.Language.fromValue(language);
Metadatum rda = getRelative(rdas, rdaMap, node.getNumbering());
Metadatum rda = getRelative(rdas, rdaMap, entry.getKey());
if (rda != null) {
rda.setLanguage(lang);
rda.setAdditionalProperty("languageId", node.getId());
@ -82,7 +83,7 @@ public class MetadataRDAMapper{
}
}
}
}
}

10
core/src/main/java/org/opencdmp/filetransformer/rda/model/rda/mapper/SecurityAndPrivacyRDAMapper.java
View File

@ -10,13 +10,14 @@ import java.util.stream.Collectors;
@Component
public class SecurityAndPrivacyRDAMapper {
public List<SecurityAndPrivacy> toRDA(List<FieldModel> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields) {
public List<SecurityAndPrivacy> toRDA(Map<String, List<FieldModel>> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields) {
if (nodes == null) return null;
if (valueFields == null) throw new IllegalArgumentException("valueFields is missing");
Map<String, SecurityAndPrivacy> rdaMap = new HashMap<>();
for (FieldModel node: nodes) {
for (Map.Entry<String, List<FieldModel>> entry: nodes.entrySet()) {
for (FieldModel node : entry.getValue()) {
String rdaProperty = node.getSchematics().stream().filter(schematic -> schematic.startsWith("rda.dataset.security_and_privacy")).findFirst().orElse("");
if (node.getData() == null) {
continue;
@ -25,9 +26,9 @@ public class SecurityAndPrivacyRDAMapper {
if (rdaValue == null || rdaValue.getTextValue() == null || rdaValue.getTextValue().isBlank()) {
continue;
}
SecurityAndPrivacy rda = getRelative(rdaMap, node.getNumbering());
SecurityAndPrivacy rda = getRelative(rdaMap, entry.getKey());
if (!rdaMap.containsValue(rda)) {
rdaMap.put(node.getNumbering(), rda);
rdaMap.put(entry.getKey(), rda);
}
for (ExportPropertyName exportPropertyName : ExportPropertyName.values()) {
if (rdaProperty.contains(exportPropertyName.getName())) {
@ -44,6 +45,7 @@ public class SecurityAndPrivacyRDAMapper {
}
}
}
}
return rdaMap.values().stream()
.filter(sap -> sap.getTitle() != null)

11
core/src/main/java/org/opencdmp/filetransformer/rda/model/rda/mapper/TechnicalResourceRDAMapper.java
View File

@ -11,12 +11,13 @@ import java.util.stream.Collectors;
@Component
public class TechnicalResourceRDAMapper {
public List<TechnicalResource> toRDA(List<FieldModel> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields ) {
public List<TechnicalResource> toRDA(Map<String, List<FieldModel>> nodes, List<org.opencdmp.commonmodels.models.description.FieldModel> valueFields ) {
if (nodes == null) return null;
if (valueFields == null) throw new IllegalArgumentException("valueFields is missing");
Map<String, TechnicalResource> rdaMap = new HashMap<>();
for (FieldModel node: nodes) {
for (Map.Entry<String, List<FieldModel>> entry: nodes.entrySet()) {
for (FieldModel node : entry.getValue()) {
String rdaProperty = node.getSchematics().stream().filter(schematic -> schematic.startsWith("rda.dataset.technical_resource")).findFirst().orElse("");
if (node.getData() == null) {
continue;
@ -25,9 +26,9 @@ public class TechnicalResourceRDAMapper {
if (rdaValue == null || rdaValue.getTextValue() == null || rdaValue.getTextValue().isBlank()) {
continue;
}
TechnicalResource rda = getRelative(rdaMap, node.getNumbering());
TechnicalResource rda = getRelative(rdaMap, entry.getKey());
if (!rdaMap.containsValue(rda)) {
rdaMap.put(node.getNumbering(), rda);
rdaMap.put(entry.getKey(), rda);
}
for (ExportPropertyName exportPropertyName : ExportPropertyName.values()) {
if (rdaProperty.contains(exportPropertyName.getName())) {
@ -44,7 +45,7 @@ public class TechnicalResourceRDAMapper {
}
}
}
}
return rdaMap.values().stream()
.filter(tr -> tr.getName() != null)
.collect(Collectors.toList());

5
core/src/main/java/org/opencdmp/filetransformer/rda/service/descriptiontemplatesearcher/TemplateFieldSearcherService.java
View File

@ -4,9 +4,10 @@ import org.opencdmp.commonmodels.models.descriptiotemplate.DescriptionTemplateMo
import org.opencdmp.commonmodels.models.descriptiotemplate.FieldModel;
import java.util.List;
import java.util.Map;
public interface TemplateFieldSearcherService {
List<FieldModel> searchFieldsById(DescriptionTemplateModel template, String value);
Map<String, List<FieldModel>> searchFieldsById(DescriptionTemplateModel template, String value);
List<FieldModel> searchFieldsBySemantics(DescriptionTemplateModel template, String value);
Map<String, List<FieldModel>> searchFieldsBySemantics(DescriptionTemplateModel template, String value);
}

75
core/src/main/java/org/opencdmp/filetransformer/rda/service/descriptiontemplatesearcher/TemplateFieldSearcherServiceImpl.java
View File

@ -3,23 +3,80 @@ package org.opencdmp.filetransformer.rda.service.descriptiontemplatesearcher;
import org.opencdmp.commonmodels.models.descriptiotemplate.*;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.*;
import java.util.stream.Collectors;
@Component
public class TemplateFieldSearcherServiceImpl implements TemplateFieldSearcherService {
@Override
public List<FieldModel> searchFieldsById(DescriptionTemplateModel template, String value) {
if (template == null || template.getDefinition() == null) return new ArrayList<>();
return template.getDefinition().getFieldById(value);
public Map<String, List<FieldModel>> searchFieldsById(DescriptionTemplateModel template, String value) {
if (template == null || template.getDefinition() == null) return new HashMap<>();
List<FieldModel> fieldModels = template.getDefinition().getFieldById(value);
Map<String, List<FieldModel>> response = new HashMap<>();
for (FieldModel fieldModel : fieldModels){
String key = this.buildFieldKey(template, fieldModel);
if (!response.containsKey(key)) response.put(key, new ArrayList<>());
response.get(key).add(fieldModel);
}
return response;
}
@Override
public List<FieldModel> searchFieldsBySemantics(DescriptionTemplateModel template, String value) {
if (template == null || template.getDefinition() == null) return new ArrayList<>();
public Map<String, List<FieldModel>> searchFieldsBySemantics(DescriptionTemplateModel template, String value) {
if (template == null || template.getDefinition() == null) return new HashMap<>();
List<FieldModel> fieldModels = template.getDefinition().getAllField();
if (fieldModels == null) return new ArrayList<>();
return fieldModels.stream().filter(x-> x.getSchematics() != null && x.getSchematics().contains(value)).toList();
if (fieldModels == null) return new HashMap<>();
fieldModels = fieldModels.stream().filter(x-> x.getSchematics() != null && x.getSchematics().contains(value)).toList();
Map<String, List<FieldModel>> response = new HashMap<>();
for (FieldModel fieldModel : fieldModels){
String key = this.buildFieldKey(template, fieldModel);
if (!response.containsKey(key)) response.put(key, new ArrayList<>());
response.get(key).add(fieldModel);
}
return response;
}
private String buildFieldKey(DescriptionTemplateModel template, FieldModel fieldModel){
if (template == null || template.getDefinition() == null || template.getDefinition().getPages() == null) return null;
for (PageModel pageModel : template.getDefinition().getPages().stream().sorted(Comparator.comparing(PageModel::getOrdinal)).toList()){
String key = this.buildFieldKey(pageModel, fieldModel);
if (key != null) return pageModel.getOrdinal() + "." + key;
}
return null;
}
private String buildFieldKey(PageModel pageModel, FieldModel fieldModel){
if (pageModel == null || pageModel.getSections() == null) return null;
for (SectionModel sectionModel : pageModel.getSections().stream().sorted(Comparator.comparing(SectionModel::getOrdinal)).toList()){
String key = this.buildFieldKey(sectionModel, fieldModel);
if (key != null) return sectionModel.getOrdinal() + "." + key;
}
return null;
}
private String buildFieldKey(SectionModel sectionModel, FieldModel fieldModel){
if (sectionModel == null) return null;
if (sectionModel.getSections() != null) {
for (SectionModel innerSectionModel : sectionModel.getSections().stream().sorted(Comparator.comparing(SectionModel::getOrdinal)).toList()) {
String key = this.buildFieldKey(innerSectionModel, fieldModel);
if (key != null) return innerSectionModel.getOrdinal() + "." + key;
}
}
if (sectionModel.getFieldSets() != null) {
for (FieldSetModel fieldSetModel : sectionModel.getFieldSets().stream().sorted(Comparator.comparing(FieldSetModel::getOrdinal)).toList()) {
String key = this.buildFieldKey(fieldSetModel, fieldModel);
if (key != null) return fieldSetModel.getOrdinal() + "." + key;
}
}
return null;
}
private String buildFieldKey(FieldSetModel fieldSetModel, FieldModel fieldModel){
if (fieldSetModel == null || fieldSetModel.getFields() == null) return null;
for (FieldModel currentFieldModel : fieldSetModel.getFields().stream().sorted(Comparator.comparing(FieldModel::getOrdinal)).toList()){
if (currentFieldModel.getId().equals(fieldModel.getId())) return currentFieldModel.getOrdinal().toString();
}
return null;
}
}

2
pom.xml
View File

@ -30,7 +30,7 @@
<dependency>
<groupId>org.opencdmp</groupId>
<artifactId>file-transformer-base</artifactId>
<version>0.0.15</version>
<version>0.0.18</version>
</dependency>
</dependencies>