MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration fixes

This commit is contained in:
Efstratios Giannopoulos 2024-05-08 16:08:19 +03:00
parent a7efe9c80e
commit 0fa3b1ef72
5 changed files with 64 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
backend/core/src/main/java/org/opencdmp/elastic/elasticbuilder/nested/NestedCollaboratorElasticBuilder.java
View File

@ -39,8 +39,8 @@ public class NestedCollaboratorElasticBuilder extends BaseElasticBuilder<NestedC
return new ArrayList<>();
List<NestedCollaboratorElasticEntity> models = new ArrayList<>();
Map<UUID, String> userNameMap = this.queryFactory.query(UserQuery.class).disableTracking().isActive(IsActive.Active).ids(data.stream().map(DmpUserEntity::getUserId).distinct().collect(Collectors.toList())).collectAs(new BaseFieldSet().ensure(User._id).ensure(User._name))
.stream().collect(Collectors.toMap(UserEntity::getId, UserEntity::getName));
List<UserEntity> users = this.queryFactory.query(UserQuery.class).disableTracking().isActive(IsActive.Active).ids(data.stream().map(DmpUserEntity::getUserId).distinct().collect(Collectors.toList())).collectAs(new BaseFieldSet().ensure(User._id).ensure(User._name));
Map<UUID, String> userNameMap = users == null ? new HashMap<>() : users.stream().collect(Collectors.toMap(UserEntity::getId, UserEntity::getName));
for (DmpUserEntity d : data) {
NestedCollaboratorElasticEntity m = new NestedCollaboratorElasticEntity();
m.setId(d.getId());

2
dmp-frontend/src/app/ui/description/editor/description-base-fields-editor/description-base-fields-editor.component.html
View File

@ -63,4 +63,4 @@
</div>
</div>
</div>
</form>
</form>

5
dmp-frontend/src/app/ui/description/editor/description-form/visibility-rules/visibility-rules.service.ts
View File

@ -278,8 +278,11 @@ export class VisibilityRulesService {
return field.textValue == rule.textValue;
}
}
else if ([DescriptionTemplateFieldType.SELECT].includes(fieldType) && field.textValue != null && field.textValue.length > 0) {
return rule.textValue != null && rule.textValue.length > 0 && field.textValue == rule.textValue;
}
else if ([DescriptionTemplateFieldType.SELECT].includes(fieldType) && field.textListValue != null && field.textListValue.length > 0) {
return rule.textValue != null && rule.textValue.length > 0 && field.textListValue.includes(rule.textValue);
return field.textListValue.includes(rule.textValue);
}
else if ([DescriptionTemplateFieldType.REFERENCE_TYPES, DescriptionTemplateFieldType.UPLOAD, DescriptionTemplateFieldType.TAGS, DescriptionTemplateFieldType.INTERNAL_ENTRIES_DMPS,
DescriptionTemplateFieldType.INTERNAL_ENTRIES_DESCRIPTIONS, DescriptionTemplateFieldType.VALIDATION, DescriptionTemplateFieldType.DATASET_IDENTIFIER].includes(fieldType)) {

48
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -13,7 +13,6 @@ import org.opencdmp.commons.types.referencetype.ReferenceTypeDefinitionEntity;
import org.opencdmp.convention.ConventionService;
import org.opencdmp.data.*;
import org.opencdmp.model.dmp.Dmp;
import org.opencdmp.model.persist.descriptionreference.DescriptionReferenceDataPersist;
import org.opencdmp.model.reference.Reference;
import org.opencdmp.model.referencetype.ReferenceType;
import org.opencdmp.query.*;
@ -53,6 +52,7 @@ public class DatasetMigrationService {
private final XmlHandlingService xmlHandlingService;
private static final int PageSize = 500;
private static final boolean TestMode = false;
private static final int SimpleFieldSetOrdinal = -512;
private final EntityManager entityManager;
private final ConventionService conventionService;
private final MigrationTools migrationTools;
@ -197,22 +197,23 @@ public class DatasetMigrationService {
List<FieldSetEntity> fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets();
for (String key : properties.keySet()) {
if (key.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){
if (properties.get(key) != null && this.conventionService.isNullOrEmpty(properties.get(key).toString())) { continue; }
FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null);
String finalKey = key.trim();
if (finalKey.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){
if (properties.get(finalKey) != null && this.conventionService.isNullOrEmpty(properties.get(finalKey).toString())) { continue; }
FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().trim().equals(finalKey.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null);
if (currentFieldSet == null) {
logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set " + key + " not found for comment " + properties.get(key).toString());
logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set " + finalKey + " not found for comment " + properties.get(finalKey).toString());
continue;
}
this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(key).toString());
this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(finalKey).toString());
} else {
if (!key.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) {
if (!finalKey.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) {
FieldSetEntity currentFieldSet = null;
org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField = null;
for (FieldSetEntity fieldSetEntity : fieldSetEntities) {
List<org.opencdmp.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(key.trim());
List<org.opencdmp.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(finalKey.trim());
if (!fieldEntities.isEmpty()) {
currentFieldSet = fieldSetEntity;
currentField = fieldEntities.getFirst();
@ -220,7 +221,7 @@ public class DatasetMigrationService {
}
if (currentField == null) {
logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + key + " " + properties.get(key).toString());
logger.error("Migrate Dataset " + item.getId() + " " + item.getLabel() + " Field set not found for field " + finalKey + " " + properties.get(finalKey).toString());
continue;
}
@ -233,11 +234,16 @@ public class DatasetMigrationService {
for (PropertyDefinitionFieldSetEntity fieldSetEntity : propertyDefinitionEntity.getFieldSets().values()){
int newOrdinal = 0;
for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() >= 0).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){
PropertyDefinitionFieldSetItemEntity firstFieldSet = fieldSetEntity.getItems().stream().findFirst().filter(x-> x.getOrdinal() == SimpleFieldSetOrdinal).orElse(null);
if (firstFieldSet != null){
firstFieldSet.setOrdinal(newOrdinal);
newOrdinal++;
}
for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() >= 0 && x.getOrdinal() != SimpleFieldSetOrdinal).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){
propertyDefinitionFieldSetItemEntity.setOrdinal(newOrdinal);
newOrdinal++;
}
for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() < 0).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){
for (PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity : fieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() < 0 && x.getOrdinal() != SimpleFieldSetOrdinal).sorted(Comparator.comparingInt(PropertyDefinitionFieldSetItemEntity::getOrdinal)).toList()){
propertyDefinitionFieldSetItemEntity.setOrdinal(newOrdinal);
newOrdinal++;
}
@ -306,6 +312,8 @@ public class DatasetMigrationService {
} else {
if (groupOrdinalMapping.get(new MultipleFieldKey(fieldSetId, groupId)) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key);
}
if (ordinal == SimpleFieldSetOrdinal) throw new MyApplicationException("Fieldset has migration simple fieldset ordinal " + SimpleFieldSetOrdinal);
this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
@ -323,44 +331,44 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
}
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
}
private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null);
if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity);
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId().trim(), propertyDefinitionFieldSetEntity);
}
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null;
if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
propertyDefinitionFieldSetItemEntity.setOrdinal(0);
propertyDefinitionFieldSetItemEntity.setOrdinal(SimpleFieldSetOrdinal);
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
} else {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
}
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
}
private void addSimpleCommentField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, String comment){
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null);
if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity);
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId().trim(), propertyDefinitionFieldSetEntity);
}
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null;
if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
propertyDefinitionFieldSetItemEntity.setOrdinal(0);
propertyDefinitionFieldSetItemEntity.setOrdinal(SimpleFieldSetOrdinal);
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
} else {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
@ -1021,7 +1029,7 @@ public class DatasetMigrationService {
private @NotNull DescriptionReferenceDataEntity buildDescriptionReferenceDataEntity(org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField){
DescriptionReferenceDataEntity data = new DescriptionReferenceDataEntity();
data.setFieldId(currentField.getId());
data.setFieldId(currentField.getId().trim());
return data;
}

60
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java
View File

@ -1,23 +1,8 @@
package eu.old.eudat.migration;
import com.fasterxml.jackson.annotation.JsonValue;
import eu.old.eudat.models.data.components.commons.DefaultValue;
import org.opencdmp.commons.XmlHandlingService;
import org.opencdmp.commons.enums.*;
import org.opencdmp.commons.types.descriptiontemplate.*;
import org.opencdmp.commons.types.descriptiontemplate.fielddata.*;
import org.opencdmp.commons.types.externalfetcher.*;
import org.opencdmp.commons.types.referencetype.*;
import org.opencdmp.convention.ConventionService;
import org.opencdmp.data.DescriptionTemplateEntity;
import org.opencdmp.data.ReferenceEntity;
import org.opencdmp.data.ReferenceTypeEntity;
import org.opencdmp.data.converters.enums.DatabaseEnum;
import org.opencdmp.model.descriptiontemplate.DescriptionTemplate;
import org.opencdmp.query.DescriptionTemplateQuery;
import org.opencdmp.query.ReferenceTypeQuery;
import eu.old.eudat.logic.services.helpers.HelpersService;
import eu.old.eudat.logic.utilities.builders.XmlBuilder;
import eu.old.eudat.models.data.components.commons.DefaultValue;
import eu.old.eudat.models.data.components.commons.Multiplicity;
import eu.old.eudat.models.data.components.commons.Rule;
import eu.old.eudat.models.data.components.commons.datafield.*;
@ -31,8 +16,21 @@ import jakarta.persistence.EntityManager;
import jakarta.ws.rs.NotSupportedException;
import jakarta.xml.bind.JAXBException;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.common.inject.Guice;
import org.jetbrains.annotations.NotNull;
import org.opencdmp.commons.XmlHandlingService;
import org.opencdmp.commons.enums.*;
import org.opencdmp.commons.types.descriptiontemplate.*;
import org.opencdmp.commons.types.descriptiontemplate.fielddata.*;
import org.opencdmp.commons.types.externalfetcher.*;
import org.opencdmp.commons.types.referencetype.ReferenceTypeDefinitionEntity;
import org.opencdmp.convention.ConventionService;
import org.opencdmp.data.DescriptionTemplateEntity;
import org.opencdmp.data.ReferenceEntity;
import org.opencdmp.data.ReferenceTypeEntity;
import org.opencdmp.data.converters.enums.DatabaseEnum;
import org.opencdmp.model.descriptiontemplate.DescriptionTemplate;
import org.opencdmp.query.DescriptionTemplateQuery;
import org.opencdmp.query.ReferenceTypeQuery;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
@ -128,7 +126,7 @@ public class DescriptionTemplateXmlMigrationService {
Map<String, List<Section>> sectionPerPage = mapSectionsToPage(persist);
data.setPages(new ArrayList<>());
for (Page pagePersist : persist.getPages()) {
data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId()), referenceTypeEntityMap, knownReferenceEntities));
data.getPages().add(this.buildPageEntity(pagePersist, sectionPerPage.get(pagePersist.getId().trim()), referenceTypeEntityMap, knownReferenceEntities));
}
}
@ -140,7 +138,7 @@ public class DescriptionTemplateXmlMigrationService {
if (persist == null)
return data;
data.setId(persist.getId());
data.setId(persist.getId().trim());
data.setDescription(persist.getDescription());
data.setExtendedDescription(persist.getExtendedDescription());
data.setOrdinal(persist.getOrdinal());
@ -168,7 +166,7 @@ public class DescriptionTemplateXmlMigrationService {
if (persist == null)
return data;
data.setId(persist.getId());
data.setId(persist.getId().trim());
data.setOrdinal(persist.getOrdinal());
data.setTitle(persist.getTitle());
data.setDescription(persist.getDescription());
@ -193,7 +191,7 @@ public class DescriptionTemplateXmlMigrationService {
if (persist == null)
return data;
data.setId(persist.getId());
data.setId(persist.getId().trim());
data.setOrdinal(persist.getOrdinal());
data.setSchematics(persist.getSchematics());
if (persist.getValidations() != null) data.setValidations(persist.getValidations().stream().map(x-> {
@ -455,7 +453,7 @@ public class DescriptionTemplateXmlMigrationService {
String textValue = persist.getValue();
if (textValue == null || textValue.isEmpty()) return data;
data.setTarget(persist.getTarget());
data.setTarget(persist.getTarget().trim());
switch (fieldType){
case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> data.setTextValue(textValue.trim());
case CHECK_BOX, BOOLEAN_DECISION -> {
@ -477,7 +475,7 @@ public class DescriptionTemplateXmlMigrationService {
case SELECT -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
String[] valuesParsed = migrationTools.tryParseJsonAsObjectString(String[].class, textValue);
data.setTextValue(valuesParsed != null && valuesParsed.length > 0 ? valuesParsed[0] : null);
data.setTextValue(valuesParsed != null && valuesParsed.length > 0 ? valuesParsed[0] : textValue);
}
}
case DATASET_IDENTIFIER -> {
@ -519,7 +517,7 @@ public class DescriptionTemplateXmlMigrationService {
if (persist == null)
return data;
data.setId(persist.getId());
data.setId(persist.getId().trim());
data.setOrdinal(persist.getOrdinal());
data.setTitle(persist.getTitle());
@ -539,18 +537,18 @@ public class DescriptionTemplateXmlMigrationService {
if (persist == null || persist.getPages() == null || persist.getSections() == null) return sectionPerPage;
for (Page page :persist.getPages()){
sectionPerPage.put(page.getId(), new ArrayList<>());
sectionPerPage.put(page.getId().trim(), new ArrayList<>());
for (Section section : persist.getSections()){
if (this.isInPage(section, page)){
if (mappedSectionsIds.contains(section.getId())) throw new MyApplicationException("Multiple page section found for page " + page.getId() + " section " + section.getId());
mappedSectionsIds.add(section.getId());
sectionPerPage.get(page.getId()).add(section);
if (mappedSectionsIds.contains(section.getId().trim())) throw new MyApplicationException("Multiple page section found for page " + page.getId().trim() + " section " + section.getId().trim());
mappedSectionsIds.add(section.getId().trim());
sectionPerPage.get(page.getId().trim()).add(section);
}
}
}
for (Section section : persist.getSections()){
if (!mappedSectionsIds.contains(section.getId())) throw new MyApplicationException("Orphan section found " + section.getId());
if (!mappedSectionsIds.contains(section.getId().trim())) throw new MyApplicationException("Orphan section found " + section.getId().trim());
}
return sectionPerPage;
@ -558,9 +556,9 @@ public class DescriptionTemplateXmlMigrationService {
private boolean isInPage(Section section, Page page){
try {
return UUID.fromString(section.getPage()).equals(UUID.fromString(page.getId()));
return UUID.fromString(section.getPage()).equals(UUID.fromString(page.getId().trim()));
} catch (Exception e){
return section.getPage().toLowerCase(Locale.ROOT).equals(page.getId().toLowerCase(Locale.ROOT));
return section.getPage().toLowerCase(Locale.ROOT).equals(page.getId().trim().toLowerCase(Locale.ROOT));
}
}