From 0fa3b1ef72b91a57c07f60ae7f749c4c29ccc25a Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Wed, 8 May 2024 16:08:19 +0300 Subject: [PATCH] migration fixes --- .../NestedCollaboratorElasticBuilder.java | 4 +- ...cription-base-fields-editor.component.html | 2 +- .../visibility-rules.service.ts | 5 +- .../migration/DatasetMigrationService.java | 48 ++++++++------- ...escriptionTemplateXmlMigrationService.java | 60 +++++++++---------- 5 files changed, 64 insertions(+), 55 deletions(-) diff --git a/backend/core/src/main/java/org/opencdmp/elastic/elasticbuilder/nested/NestedCollaboratorElasticBuilder.java b/backend/core/src/main/java/org/opencdmp/elastic/elasticbuilder/nested/NestedCollaboratorElasticBuilder.java index 1af366104..bb4d0622d 100644 --- a/backend/core/src/main/java/org/opencdmp/elastic/elasticbuilder/nested/NestedCollaboratorElasticBuilder.java +++ b/backend/core/src/main/java/org/opencdmp/elastic/elasticbuilder/nested/NestedCollaboratorElasticBuilder.java @@ -39,8 +39,8 @@ public class NestedCollaboratorElasticBuilder extends BaseElasticBuilder(); List models = new ArrayList<>(); - Map userNameMap = this.queryFactory.query(UserQuery.class).disableTracking().isActive(IsActive.Active).ids(data.stream().map(DmpUserEntity::getUserId).distinct().collect(Collectors.toList())).collectAs(new BaseFieldSet().ensure(User._id).ensure(User._name)) - .stream().collect(Collectors.toMap(UserEntity::getId, UserEntity::getName)); + List users = this.queryFactory.query(UserQuery.class).disableTracking().isActive(IsActive.Active).ids(data.stream().map(DmpUserEntity::getUserId).distinct().collect(Collectors.toList())).collectAs(new BaseFieldSet().ensure(User._id).ensure(User._name)); + Map userNameMap = users == null ? new HashMap<>() : users.stream().collect(Collectors.toMap(UserEntity::getId, UserEntity::getName)); for (DmpUserEntity d : data) { NestedCollaboratorElasticEntity m = new NestedCollaboratorElasticEntity(); m.setId(d.getId()); diff --git a/dmp-frontend/src/app/ui/description/editor/description-base-fields-editor/description-base-fields-editor.component.html b/dmp-frontend/src/app/ui/description/editor/description-base-fields-editor/description-base-fields-editor.component.html index 4b6a91267..f92d3df39 100644 --- a/dmp-frontend/src/app/ui/description/editor/description-base-fields-editor/description-base-fields-editor.component.html +++ b/dmp-frontend/src/app/ui/description/editor/description-base-fields-editor/description-base-fields-editor.component.html @@ -63,4 +63,4 @@ - \ No newline at end of file + diff --git a/dmp-frontend/src/app/ui/description/editor/description-form/visibility-rules/visibility-rules.service.ts b/dmp-frontend/src/app/ui/description/editor/description-form/visibility-rules/visibility-rules.service.ts index 8ebea87e0..c427aaf2f 100644 --- a/dmp-frontend/src/app/ui/description/editor/description-form/visibility-rules/visibility-rules.service.ts +++ b/dmp-frontend/src/app/ui/description/editor/description-form/visibility-rules/visibility-rules.service.ts @@ -278,8 +278,11 @@ export class VisibilityRulesService { return field.textValue == rule.textValue; } } + else if ([DescriptionTemplateFieldType.SELECT].includes(fieldType) && field.textValue != null && field.textValue.length > 0) { + return rule.textValue != null && rule.textValue.length > 0 && field.textValue == rule.textValue; + } else if ([DescriptionTemplateFieldType.SELECT].includes(fieldType) && field.textListValue != null && field.textListValue.length > 0) { - return rule.textValue != null && rule.textValue.length > 0 && field.textListValue.includes(rule.textValue); + return field.textListValue.includes(rule.textValue); } else if ([DescriptionTemplateFieldType.REFERENCE_TYPES, DescriptionTemplateFieldType.UPLOAD, DescriptionTemplateFieldType.TAGS, DescriptionTemplateFieldType.INTERNAL_ENTRIES_DMPS, DescriptionTemplateFieldType.INTERNAL_ENTRIES_DESCRIPTIONS, DescriptionTemplateFieldType.VALIDATION, DescriptionTemplateFieldType.DATASET_IDENTIFIER].includes(fieldType)) { diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java index 57e8f95dd..b6699a062 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java @@ -13,7 +13,6 @@ import org.opencdmp.commons.types.referencetype.ReferenceTypeDefinitionEntity; import org.opencdmp.convention.ConventionService; import org.opencdmp.data.*; import org.opencdmp.model.dmp.Dmp; -import org.opencdmp.model.persist.descriptionreference.DescriptionReferenceDataPersist; import org.opencdmp.model.reference.Reference; import org.opencdmp.model.referencetype.ReferenceType; import org.opencdmp.query.*; @@ -53,6 +52,7 @@ public class DatasetMigrationService { private final XmlHandlingService xmlHandlingService; private static final int PageSize = 500; private static final boolean TestMode = false; + private static final int SimpleFieldSetOrdinal = -512; private final EntityManager entityManager; private final ConventionService conventionService; private final MigrationTools migrationTools; @@ -197,22 +197,23 @@ public class DatasetMigrationService { List fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets(); for (String key : properties.keySet()) { - if (key.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){ - if (properties.get(key) != null && this.conventionService.isNullOrEmpty(properties.get(key).toString())) { continue; } - FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null); + String finalKey = key.trim(); + if (finalKey.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){ + if (properties.get(finalKey) != null && this.conventionService.isNullOrEmpty(properties.get(finalKey).toString())) { continue; } + FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().trim().equals(finalKey.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null); if (currentFieldSet == null) { - logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set " + key + " not found for comment " + properties.get(key).toString()); + logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set " + finalKey + " not found for comment " + properties.get(finalKey).toString()); continue; } - this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(key).toString()); + this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(finalKey).toString()); } else { - if (!key.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) { + if (!finalKey.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) { FieldSetEntity currentFieldSet = null; org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField = null; for (FieldSetEntity fieldSetEntity : fieldSetEntities) { - List fieldEntities = fieldSetEntity.getFieldById(key.trim()); + List fieldEntities = fieldSetEntity.getFieldById(finalKey.trim()); if (!fieldEntities.isEmpty()) { currentFieldSet = fieldSetEntity; currentField = fieldEntities.getFirst(); @@ -220,7 +221,7 @@ public class DatasetMigrationService { } if (currentField == null) { - logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + key + " " + properties.get(key).toString()); + logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + finalKey + " " + properties.get(finalKey).toString()); continue; } @@ -233,11 +234,16 @@ public class DatasetMigrationService { for (PropertyDefinitionFieldSetEntity fieldSetEntity : propertyDefinitionEntity.getFieldSets().values()){ int newOrdinal = 0; - for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() >= 0).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){ + PropertyDefinitionFieldSetItemEntity firstFieldSet = fieldSetEntity.getItems().stream().findFirst().filter(x-> x.getOrdinal() == SimpleFieldSetOrdinal).orElse(null); + if (firstFieldSet != null){ + firstFieldSet.setOrdinal(newOrdinal); + newOrdinal++; + } + for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() >= 0 && x.getOrdinal() != SimpleFieldSetOrdinal).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){ propertyDefinitionFieldSetItemEntity.setOrdinal(newOrdinal); newOrdinal++; } - for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() < 0).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){ + for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() < 0 && x.getOrdinal() != SimpleFieldSetOrdinal).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){ propertyDefinitionFieldSetItemEntity.setOrdinal(newOrdinal); newOrdinal++; } @@ -306,6 +312,8 @@ public class DatasetMigrationService { } else { if (groupOrdinalMapping.get(new MultipleFieldKey(fieldSetId, groupId)) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key); } + + if (ordinal == SimpleFieldSetOrdinal) throw new MyApplicationException("Fieldset has migration simple fieldset ordinal " + SimpleFieldSetOrdinal); this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); @@ -323,44 +331,44 @@ public class DatasetMigrationService { propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); } - propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); + propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); } private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, Map referenceTypeDefinitionEntityMap, Map referenceMap){ - PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null); + PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null); if (propertyDefinitionFieldSetEntity == null) { propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); propertyDefinitionFieldSetEntity.setItems(new ArrayList<>()); - propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity); + propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId().trim(), propertyDefinitionFieldSetEntity); } PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null; if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){ propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity(); propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>()); - propertyDefinitionFieldSetItemEntity.setOrdinal(0); + propertyDefinitionFieldSetItemEntity.setOrdinal(SimpleFieldSetOrdinal); propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); } else { propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); } - propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); + propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); } private void addSimpleCommentField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, String comment){ - PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null); + PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null); if (propertyDefinitionFieldSetEntity == null) { propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); propertyDefinitionFieldSetEntity.setItems(new ArrayList<>()); - propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity); + propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId().trim(), propertyDefinitionFieldSetEntity); } PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null; if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){ propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity(); propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>()); - propertyDefinitionFieldSetItemEntity.setOrdinal(0); + propertyDefinitionFieldSetItemEntity.setOrdinal(SimpleFieldSetOrdinal); propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); } else { propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); @@ -1021,7 +1029,7 @@ public class DatasetMigrationService { private @NotNull DescriptionReferenceDataEntity buildDescriptionReferenceDataEntity(org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField){ DescriptionReferenceDataEntity data = new DescriptionReferenceDataEntity(); - data.setFieldId(currentField.getId()); + data.setFieldId(currentField.getId().trim()); return data; } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java index c52dc900e..3ba162d43 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java @@ -1,23 +1,8 @@ package eu.old.eudat.migration; import com.fasterxml.jackson.annotation.JsonValue; -import eu.old.eudat.models.data.components.commons.DefaultValue; -import org.opencdmp.commons.XmlHandlingService; -import org.opencdmp.commons.enums.*; -import org.opencdmp.commons.types.descriptiontemplate.*; -import org.opencdmp.commons.types.descriptiontemplate.fielddata.*; -import org.opencdmp.commons.types.externalfetcher.*; -import org.opencdmp.commons.types.referencetype.*; -import org.opencdmp.convention.ConventionService; -import org.opencdmp.data.DescriptionTemplateEntity; -import org.opencdmp.data.ReferenceEntity; -import org.opencdmp.data.ReferenceTypeEntity; -import org.opencdmp.data.converters.enums.DatabaseEnum; -import org.opencdmp.model.descriptiontemplate.DescriptionTemplate; -import org.opencdmp.query.DescriptionTemplateQuery; -import org.opencdmp.query.ReferenceTypeQuery; -import eu.old.eudat.logic.services.helpers.HelpersService; import eu.old.eudat.logic.utilities.builders.XmlBuilder; +import eu.old.eudat.models.data.components.commons.DefaultValue; import eu.old.eudat.models.data.components.commons.Multiplicity; import eu.old.eudat.models.data.components.commons.Rule; import eu.old.eudat.models.data.components.commons.datafield.*; @@ -31,8 +16,21 @@ import jakarta.persistence.EntityManager; import jakarta.ws.rs.NotSupportedException; import jakarta.xml.bind.JAXBException; import org.apache.commons.lang3.StringUtils; -import org.elasticsearch.common.inject.Guice; import org.jetbrains.annotations.NotNull; +import org.opencdmp.commons.XmlHandlingService; +import org.opencdmp.commons.enums.*; +import org.opencdmp.commons.types.descriptiontemplate.*; +import org.opencdmp.commons.types.descriptiontemplate.fielddata.*; +import org.opencdmp.commons.types.externalfetcher.*; +import org.opencdmp.commons.types.referencetype.ReferenceTypeDefinitionEntity; +import org.opencdmp.convention.ConventionService; +import org.opencdmp.data.DescriptionTemplateEntity; +import org.opencdmp.data.ReferenceEntity; +import org.opencdmp.data.ReferenceTypeEntity; +import org.opencdmp.data.converters.enums.DatabaseEnum; +import org.opencdmp.model.descriptiontemplate.DescriptionTemplate; +import org.opencdmp.query.DescriptionTemplateQuery; +import org.opencdmp.query.ReferenceTypeQuery; import org.slf4j.LoggerFactory; import org.springframework.http.MediaType; import org.springframework.stereotype.Service; @@ -128,7 +126,7 @@ public class DescriptionTemplateXmlMigrationService { Map> sectionPerPage = mapSectionsToPage(persist); data.setPages(new ArrayList<>()); for (Page pagePersist : persist.getPages()) { - data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId()), referenceTypeEntityMap, knownReferenceEntities)); + data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId().trim()), referenceTypeEntityMap, knownReferenceEntities)); } } @@ -140,7 +138,7 @@ public class DescriptionTemplateXmlMigrationService { if (persist == null) return data; - data.setId(persist.getId()); + data.setId(persist.getId().trim()); data.setDescription(persist.getDescription()); data.setExtendedDescription(persist.getExtendedDescription()); data.setOrdinal(persist.getOrdinal()); @@ -168,7 +166,7 @@ public class DescriptionTemplateXmlMigrationService { if (persist == null) return data; - data.setId(persist.getId()); + data.setId(persist.getId().trim()); data.setOrdinal(persist.getOrdinal()); data.setTitle(persist.getTitle()); data.setDescription(persist.getDescription()); @@ -193,7 +191,7 @@ public class DescriptionTemplateXmlMigrationService { if (persist == null) return data; - data.setId(persist.getId()); + data.setId(persist.getId().trim()); data.setOrdinal(persist.getOrdinal()); data.setSchematics(persist.getSchematics()); if (persist.getValidations() != null) data.setValidations(persist.getValidations().stream().map(x-> { @@ -455,7 +453,7 @@ public class DescriptionTemplateXmlMigrationService { String textValue = persist.getValue(); if (textValue == null || textValue.isEmpty()) return data; - data.setTarget(persist.getTarget()); + data.setTarget(persist.getTarget().trim()); switch (fieldType){ case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> data.setTextValue(textValue.trim()); case CHECK_BOX, BOOLEAN_DECISION -> { @@ -477,7 +475,7 @@ public class DescriptionTemplateXmlMigrationService { case SELECT -> { if(!this.conventionService.isNullOrEmpty(textValue)) { String[] valuesParsed = migrationTools.tryParseJsonAsObjectString(String[].class, textValue); - data.setTextValue(valuesParsed != null && valuesParsed.length > 0 ? valuesParsed[0] : null); + data.setTextValue(valuesParsed != null && valuesParsed.length > 0 ? valuesParsed[0] : textValue); } } case DATASET_IDENTIFIER -> { @@ -519,7 +517,7 @@ public class DescriptionTemplateXmlMigrationService { if (persist == null) return data; - data.setId(persist.getId()); + data.setId(persist.getId().trim()); data.setOrdinal(persist.getOrdinal()); data.setTitle(persist.getTitle()); @@ -539,18 +537,18 @@ public class DescriptionTemplateXmlMigrationService { if (persist == null || persist.getPages() == null || persist.getSections() == null) return sectionPerPage; for (Page page :persist.getPages()){ - sectionPerPage.put(page.getId(), new ArrayList<>()); + sectionPerPage.put(page.getId().trim(), new ArrayList<>()); for (Section section : persist.getSections()){ if (this.isInPage(section, page)){ - if (mappedSectionsIds.contains(section.getId())) throw new MyApplicationException("Multiple page section found for page " + page.getId() + " section " + section.getId()); - mappedSectionsIds.add(section.getId()); - sectionPerPage.get(page.getId()).add(section); + if (mappedSectionsIds.contains(section.getId().trim())) throw new MyApplicationException("Multiple page section found for page " + page.getId().trim() + " section " + section.getId().trim()); + mappedSectionsIds.add(section.getId().trim()); + sectionPerPage.get(page.getId().trim()).add(section); } } } for (Section section : persist.getSections()){ - if (!mappedSectionsIds.contains(section.getId())) throw new MyApplicationException("Orphan section found " + section.getId()); + if (!mappedSectionsIds.contains(section.getId().trim())) throw new MyApplicationException("Orphan section found " + section.getId().trim()); } return sectionPerPage; @@ -558,9 +556,9 @@ public class DescriptionTemplateXmlMigrationService { private boolean isInPage(Section section, Page page){ try { - return UUID.fromString(section.getPage()).equals(UUID.fromString(page.getId())); + return UUID.fromString(section.getPage()).equals(UUID.fromString(page.getId().trim())); } catch (Exception e){ - return section.getPage().toLowerCase(Locale.ROOT).equals(page.getId().toLowerCase(Locale.ROOT)); + return section.getPage().toLowerCase(Locale.ROOT).equals(page.getId().trim().toLowerCase(Locale.ROOT)); } }