From e1fed4dc9d3d7fb9356ea177a93564ccd9d6faf9 Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Mon, 1 Jul 2024 16:31:42 +0300 Subject: [PATCH] migration fixes --- .../migration/DatasetMigrationService.java | 88 ++++++++++++++++--- .../eudat/migration/TagMigrationService.java | 26 +++++- 2 files changed, 99 insertions(+), 15 deletions(-) diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java index 0512e8020..78fbd6fb3 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java @@ -1,6 +1,7 @@ package eu.old.eudat.migration; import eu.old.eudat.data.entities.DMP; +import eu.old.eudat.elastic.entities.Tag; import org.opencdmp.commons.JsonHandlingService; import org.opencdmp.commons.XmlHandlingService; import org.opencdmp.commons.enums.*; @@ -86,6 +87,8 @@ public class DatasetMigrationService { for (ReferenceEntity reference : references){ if (reference.getCreatedById() == null) referenceMap.put(new ReferenceKey(reference), reference); } + Map savedTagIdsByName = new HashMap<>(); + HashSet existingTagIds = new HashSet<>(); List items; do { @@ -181,7 +184,7 @@ public class DatasetMigrationService { this.entityManager.persist(data); this.entityManager.flush(); - data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap))); + data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap, savedTagIdsByName, existingTagIds))); this.entityManager.merge(data); this.entityManager.flush(); @@ -216,7 +219,7 @@ public class DatasetMigrationService { return itemDescriptionTemplates; } - private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map referenceTypeDefinitionEntityMap, Map referenceMap) { + private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map referenceTypeDefinitionEntityMap, Map referenceMap, Map savedTagIdsByName, HashSet existingTagIds) { if (this.conventionService.isNullOrEmpty(item.getProperties())) return null; JSONObject jObject = new JSONObject(item.getProperties()); Map properties = new HashMap<>(); @@ -256,12 +259,12 @@ public class DatasetMigrationService { continue; } - this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities); + this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds); } } } - this.addMultipleField(item, properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities); + this.addMultipleField(item, properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds); this.entityManager.flush(); @@ -310,7 +313,7 @@ public class DatasetMigrationService { } } - private void addMultipleField(Dataset item, Map properties, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map referenceTypeDefinitionEntityMap, Map referenceMap, List createdDescriptionReferenceEntities){ + private void addMultipleField(Dataset item, Map properties, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map referenceTypeDefinitionEntityMap, Map referenceMap, List createdDescriptionReferenceEntities, Map savedTagIdsByName, HashSet existingTagIds){ Map groupOrdinalMapping = new HashMap<>(); for (String key : properties.keySet()) { if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) { @@ -374,13 +377,13 @@ public class DatasetMigrationService { if (ordinal == SimpleFieldSetOrdinal) throw new MyApplicationException("Fieldset has migration simple fieldset ordinal " + SimpleFieldSetOrdinal); - this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities); + this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds); } } } - private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map referenceTypeDefinitionEntityMap, Map referenceMap, List createdDescriptionReferenceEntities){ + private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map referenceTypeDefinitionEntityMap, Map referenceMap, List createdDescriptionReferenceEntities, Map savedTagIdsByName, HashSet existingTagIds){ PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null); if (propertyDefinitionFieldSetItemEntity == null){ @@ -390,10 +393,10 @@ public class DatasetMigrationService { propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); } - propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap, propertyDefinitionFieldSetItemEntity.getOrdinal(), createdDescriptionReferenceEntities)); + propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap, propertyDefinitionFieldSetItemEntity.getOrdinal(), createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds)); } - private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map referenceTypeDefinitionEntityMap, Map referenceMap, List createdDescriptionReferenceEntities){ + private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map referenceTypeDefinitionEntityMap, Map referenceMap, List createdDescriptionReferenceEntities, Map savedTagIdsByName, HashSet existingTagIds){ PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null); if (propertyDefinitionFieldSetEntity == null) { propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); @@ -411,7 +414,7 @@ public class DatasetMigrationService { propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); } - propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap, propertyDefinitionFieldSetItemEntity.getOrdinal(), createdDescriptionReferenceEntities)); + propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap, propertyDefinitionFieldSetItemEntity.getOrdinal(), createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds)); } @@ -436,7 +439,7 @@ public class DatasetMigrationService { propertyDefinitionFieldSetItemEntity.setComment(comment); } - private FieldEntity buildField(Dataset item, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map referenceTypeDefinitionEntityMap, Map referenceMap, int ordinal, List createdDescriptionReferenceEntities){ + private FieldEntity buildField(Dataset item, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map referenceTypeDefinitionEntityMap, Map referenceMap, int ordinal, List createdDescriptionReferenceEntities, Map savedTagIdsByName, HashSet existingTagIds){ FieldEntity fieldEntity = new FieldEntity(); String textValue = propertyValue != null ? propertyValue.toString() : null; if (textValue == null || textValue.isEmpty()) return fieldEntity; @@ -495,7 +498,13 @@ public class DatasetMigrationService { } if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, migrationTools.cleanTagString(textValue)); if (tags == null) throw new MyApplicationException("Could not parse tag : " + textValue); - fieldEntity.setTextListValue(Arrays.stream(tags).toList().stream().map(Tag::getName).toList()); + fieldEntity.setTextListValue(new ArrayList<>()); + for (Tag tag : tags){ + UUID id = this.getTagId(item, tag, savedTagIdsByName, existingTagIds); + if (id != null) { + fieldEntity.getTextListValue().add(id.toString()); + } + } } } @@ -1076,6 +1085,24 @@ public class DatasetMigrationService { } } + + private UUID getTagId(Dataset dataset,Tag tag, Map savedTagIdsByName, HashSet existingTagIds){ + UUID existingTagId = savedTagIdsByName.getOrDefault(new TagKey(dataset, tag), null); + if (existingTagId != null) return existingTagId; + TagEntity tagEntity = new TagEntity(); + if (!existingTagIds.contains(UUID.fromString(tag.getId()))) tagEntity.setId(UUID.fromString(tag.getId())); + else tagEntity.setId(UUID.randomUUID()); + tagEntity.setLabel(tag.getName()); + tagEntity.setCreatedAt(Instant.now()); + tagEntity.setUpdatedAt(Instant.now()); + if (dataset.getCreator() != null) tagEntity.setCreatedById(dataset.getCreator().getId()); + tagEntity.setIsActive(IsActive.Active); + savedTagIdsByName.put(new TagKey(dataset, tag), tagEntity.getId()); + this.entityManager.persist(tagEntity); + this.entityManager.flush(); + + return tagEntity.getId(); + } private DescriptionReferenceEntity createDescriptionReferenceEntity(UUID referenceId, UUID descriptionId, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, int ordinal){ DescriptionReferenceEntity data = new DescriptionReferenceEntity(); @@ -1138,6 +1165,43 @@ public class DatasetMigrationService { this.name = name; } } + + public static class TagKey { + private final UUID owner; + private final String name; + private final int hashCode; + + + public TagKey(Dataset item, Tag tag) { + this.name = tag.getName(); + if (item.getCreator() != null) this.owner = item.getCreator().getId(); + else this.owner = null; + hashCode = Objects.hash(this.owner, this.name); + } + + public UUID getOwner() { + return owner; + } + + public String getName() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + TagKey that = (TagKey) o; + return Objects.equals(owner, that.getOwner()) && Objects.equals(name, that.getName()); + } + + @Override + public int hashCode() { + return this.hashCode; + } + } public static class MultipleFieldKey{ private final String fieldSetId; diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java index b9eb6b79a..1f00b7b50 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java @@ -1,8 +1,10 @@ package eu.old.eudat.migration; +import gr.cite.tools.fieldset.BaseFieldSet; import org.opencdmp.commons.enums.IsActive; import org.opencdmp.data.DescriptionTagEntity; import org.opencdmp.data.ReferenceEntity; +import org.opencdmp.data.ReferenceTypeEntity; import org.opencdmp.data.TagEntity; import eu.old.eudat.data.dao.entities.DatasetDao; import eu.old.eudat.data.entities.Dataset; @@ -13,6 +15,9 @@ import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.logging.LoggerService; import jakarta.persistence.EntityManager; import jakarta.xml.bind.JAXBException; +import org.opencdmp.model.referencetype.ReferenceType; +import org.opencdmp.query.ReferenceTypeQuery; +import org.opencdmp.query.TagQuery; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.xml.sax.SAXException; @@ -31,11 +36,13 @@ public class TagMigrationService { private static final int PageSize = 500; private static final boolean TestMode = false; private final EntityManager entityManager; + private final QueryFactory queryFactory; - public TagMigrationService(DatabaseRepository databaseRepository, DatasetRepository datasetRepository, EntityManager entityManager) { + public TagMigrationService(DatabaseRepository databaseRepository, DatasetRepository datasetRepository, EntityManager entityManager, QueryFactory queryFactory) { this.databaseRepository = databaseRepository; this.datasetRepository = datasetRepository; this.entityManager = entityManager; + this.queryFactory = queryFactory; } public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { @@ -43,10 +50,16 @@ public class TagMigrationService { long total = datasetDao.asQueryable().count(); logger.debug("Migrate Tags for Dataset Total : " + total); int page = 0; - + Map savedTagIdsByName = new HashMap<>(); HashSet existingTagIds = new HashSet<>(); - + + List existingTags = this.queryFactory.query(TagQuery.class).collectAs(new BaseFieldSet().ensure(ReferenceType._id).ensure(ReferenceType._name)); + for (TagEntity tag : existingTags){ + existingTagIds.add(tag.getId()); + savedTagIdsByName.put(new TagKey(tag), tag.getId()); + } + List items; do { items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); @@ -112,6 +125,13 @@ public class TagMigrationService { hashCode = Objects.hash(this.owner, this.name); } + public TagKey(TagEntity tag) { + this.name = tag.getLabel(); + if (tag.getCreatedById() != null) this.owner = tag.getCreatedById(); + else this.owner = null; + hashCode = Objects.hash(this.owner, this.name); + } + public UUID getOwner() { return owner; }