MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration fixes

This commit is contained in:
Efstratios Giannopoulos 2024-07-01 16:31:42 +03:00
parent a4bf15aded
commit e1fed4dc9d
2 changed files with 99 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -1,6 +1,7 @@
package eu.old.eudat.migration; package eu.old.eudat.migration;
import eu.old.eudat.data.entities.DMP; import eu.old.eudat.data.entities.DMP;
import eu.old.eudat.elastic.entities.Tag;
import org.opencdmp.commons.JsonHandlingService; import org.opencdmp.commons.JsonHandlingService;
import org.opencdmp.commons.XmlHandlingService; import org.opencdmp.commons.XmlHandlingService;
import org.opencdmp.commons.enums.*; import org.opencdmp.commons.enums.*;
@ -86,6 +87,8 @@ public class DatasetMigrationService {
for (ReferenceEntity reference : references){ for (ReferenceEntity reference : references){
if (reference.getCreatedById() == null) referenceMap.put(new ReferenceKey(reference), reference); if (reference.getCreatedById() == null) referenceMap.put(new ReferenceKey(reference), reference);
} }
Map<TagKey, UUID> savedTagIdsByName = new HashMap<>();
HashSet<UUID> existingTagIds = new HashSet<>();
List<Dataset> items; List<Dataset> items;
do { do {
@ -181,7 +184,7 @@ public class DatasetMigrationService {
this.entityManager.persist(data); this.entityManager.persist(data);
this.entityManager.flush(); this.entityManager.flush();
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap))); data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap, savedTagIdsByName, existingTagIds)));
this.entityManager.merge(data); this.entityManager.merge(data);
this.entityManager.flush(); this.entityManager.flush();
@ -216,7 +219,7 @@ public class DatasetMigrationService {
return itemDescriptionTemplates; return itemDescriptionTemplates;
} }
private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap) { private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, Map<TagKey, UUID> savedTagIdsByName, HashSet<UUID> existingTagIds) {
if (this.conventionService.isNullOrEmpty(item.getProperties())) return null; if (this.conventionService.isNullOrEmpty(item.getProperties())) return null;
JSONObject jObject = new JSONObject(item.getProperties()); JSONObject jObject = new JSONObject(item.getProperties());
Map<String, Object> properties = new HashMap<>(); Map<String, Object> properties = new HashMap<>();
@ -256,12 +259,12 @@ public class DatasetMigrationService {
continue; continue;
} }
this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities); this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds);
} }
} }
} }
this.addMultipleField(item, properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities); this.addMultipleField(item, properties, descriptionTemplateDefinitionEntity, propertyDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds);
this.entityManager.flush(); this.entityManager.flush();
@ -310,7 +313,7 @@ public class DatasetMigrationService {
} }
} }
private void addMultipleField(Dataset item, Map<String, Object> properties, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, List<DescriptionReferenceEntity> createdDescriptionReferenceEntities){ private void addMultipleField(Dataset item, Map<String, Object> properties, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, PropertyDefinitionEntity propertyDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, List<DescriptionReferenceEntity> createdDescriptionReferenceEntities, Map<TagKey, UUID> savedTagIdsByName, HashSet<UUID> existingTagIds){
Map<MultipleFieldKey, Integer> groupOrdinalMapping = new HashMap<>(); Map<MultipleFieldKey, Integer> groupOrdinalMapping = new HashMap<>();
for (String key : properties.keySet()) { for (String key : properties.keySet()) {
if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) { if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) {
@ -374,13 +377,13 @@ public class DatasetMigrationService {
if (ordinal == SimpleFieldSetOrdinal) throw new MyApplicationException("Fieldset has migration simple fieldset ordinal " + SimpleFieldSetOrdinal); if (ordinal == SimpleFieldSetOrdinal) throw new MyApplicationException("Fieldset has migration simple fieldset ordinal " + SimpleFieldSetOrdinal);
this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities); this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap, createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds);
} }
} }
} }
private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, List<DescriptionReferenceEntity> createdDescriptionReferenceEntities){ private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, List<DescriptionReferenceEntity> createdDescriptionReferenceEntities, Map<TagKey, UUID> savedTagIdsByName, HashSet<UUID> existingTagIds){
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null); PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null);
if (propertyDefinitionFieldSetItemEntity == null){ if (propertyDefinitionFieldSetItemEntity == null){
@ -390,10 +393,10 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
} }
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap, propertyDefinitionFieldSetItemEntity.getOrdinal(), createdDescriptionReferenceEntities)); propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap, propertyDefinitionFieldSetItemEntity.getOrdinal(), createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds));
} }
private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, List<DescriptionReferenceEntity> createdDescriptionReferenceEntities){ private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, List<DescriptionReferenceEntity> createdDescriptionReferenceEntities, Map<TagKey, UUID> savedTagIdsByName, HashSet<UUID> existingTagIds){
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null); PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null);
if (propertyDefinitionFieldSetEntity == null) { if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
@ -411,7 +414,7 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
} }
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap, propertyDefinitionFieldSetItemEntity.getOrdinal(), createdDescriptionReferenceEntities)); propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap, propertyDefinitionFieldSetItemEntity.getOrdinal(), createdDescriptionReferenceEntities, savedTagIdsByName, existingTagIds));
} }
@ -436,7 +439,7 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetItemEntity.setComment(comment); propertyDefinitionFieldSetItemEntity.setComment(comment);
} }
private FieldEntity buildField(Dataset item, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, int ordinal, List<DescriptionReferenceEntity> createdDescriptionReferenceEntities){ private FieldEntity buildField(Dataset item, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap, int ordinal, List<DescriptionReferenceEntity> createdDescriptionReferenceEntities, Map<TagKey, UUID> savedTagIdsByName, HashSet<UUID> existingTagIds){
FieldEntity fieldEntity = new FieldEntity(); FieldEntity fieldEntity = new FieldEntity();
String textValue = propertyValue != null ? propertyValue.toString() : null; String textValue = propertyValue != null ? propertyValue.toString() : null;
if (textValue == null || textValue.isEmpty()) return fieldEntity; if (textValue == null || textValue.isEmpty()) return fieldEntity;
@ -495,7 +498,13 @@ public class DatasetMigrationService {
} }
if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, migrationTools.cleanTagString(textValue)); if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, migrationTools.cleanTagString(textValue));
if (tags == null) throw new MyApplicationException("Could not parse tag : " + textValue); if (tags == null) throw new MyApplicationException("Could not parse tag : " + textValue);
fieldEntity.setTextListValue(Arrays.stream(tags).toList().stream().map(Tag::getName).toList()); fieldEntity.setTextListValue(new ArrayList<>());
for (Tag tag : tags){
UUID id = this.getTagId(item, tag, savedTagIdsByName, existingTagIds);
if (id != null) {
fieldEntity.getTextListValue().add(id.toString());
}
}
} }
} }
@ -1077,6 +1086,24 @@ public class DatasetMigrationService {
} }
private UUID getTagId(Dataset dataset,Tag tag, Map<TagKey, UUID> savedTagIdsByName, HashSet<UUID> existingTagIds){
UUID existingTagId = savedTagIdsByName.getOrDefault(new TagKey(dataset, tag), null);
if (existingTagId != null) return existingTagId;
TagEntity tagEntity = new TagEntity();
if (!existingTagIds.contains(UUID.fromString(tag.getId()))) tagEntity.setId(UUID.fromString(tag.getId()));
else tagEntity.setId(UUID.randomUUID());
tagEntity.setLabel(tag.getName());
tagEntity.setCreatedAt(Instant.now());
tagEntity.setUpdatedAt(Instant.now());
if (dataset.getCreator() != null) tagEntity.setCreatedById(dataset.getCreator().getId());
tagEntity.setIsActive(IsActive.Active);
savedTagIdsByName.put(new TagKey(dataset, tag), tagEntity.getId());
this.entityManager.persist(tagEntity);
this.entityManager.flush();
return tagEntity.getId();
}
private DescriptionReferenceEntity createDescriptionReferenceEntity(UUID referenceId, UUID descriptionId, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, int ordinal){ private DescriptionReferenceEntity createDescriptionReferenceEntity(UUID referenceId, UUID descriptionId, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, int ordinal){
DescriptionReferenceEntity data = new DescriptionReferenceEntity(); DescriptionReferenceEntity data = new DescriptionReferenceEntity();
data.setId(UUID.randomUUID()); data.setId(UUID.randomUUID());
@ -1139,6 +1166,43 @@ public class DatasetMigrationService {
} }
} }
public static class TagKey {
private final UUID owner;
private final String name;
private final int hashCode;
public TagKey(Dataset item, Tag tag) {
this.name = tag.getName();
if (item.getCreator() != null) this.owner = item.getCreator().getId();
else this.owner = null;
hashCode = Objects.hash(this.owner, this.name);
}
public UUID getOwner() {
return owner;
}
public String getName() {
return name;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
TagKey that = (TagKey) o;
return Objects.equals(owner, that.getOwner()) && Objects.equals(name, that.getName());
}
@Override
public int hashCode() {
return this.hashCode;
}
}
public static class MultipleFieldKey{ public static class MultipleFieldKey{
private final String fieldSetId; private final String fieldSetId;
private final String groupKey; private final String groupKey;

22
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java
View File

@ -1,8 +1,10 @@
package eu.old.eudat.migration; package eu.old.eudat.migration;
import gr.cite.tools.fieldset.BaseFieldSet;
import org.opencdmp.commons.enums.IsActive; import org.opencdmp.commons.enums.IsActive;
import org.opencdmp.data.DescriptionTagEntity; import org.opencdmp.data.DescriptionTagEntity;
import org.opencdmp.data.ReferenceEntity; import org.opencdmp.data.ReferenceEntity;
import org.opencdmp.data.ReferenceTypeEntity;
import org.opencdmp.data.TagEntity; import org.opencdmp.data.TagEntity;
import eu.old.eudat.data.dao.entities.DatasetDao; import eu.old.eudat.data.dao.entities.DatasetDao;
import eu.old.eudat.data.entities.Dataset; import eu.old.eudat.data.entities.Dataset;
@ -13,6 +15,9 @@ import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.logging.LoggerService; import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.JAXBException;
import org.opencdmp.model.referencetype.ReferenceType;
import org.opencdmp.query.ReferenceTypeQuery;
import org.opencdmp.query.TagQuery;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.xml.sax.SAXException; import org.xml.sax.SAXException;
@ -31,11 +36,13 @@ public class TagMigrationService {
private static final int PageSize = 500; private static final int PageSize = 500;
private static final boolean TestMode = false; private static final boolean TestMode = false;
private final EntityManager entityManager; private final EntityManager entityManager;
private final QueryFactory queryFactory;
public TagMigrationService(DatabaseRepository databaseRepository, DatasetRepository datasetRepository, EntityManager entityManager) { public TagMigrationService(DatabaseRepository databaseRepository, DatasetRepository datasetRepository, EntityManager entityManager, QueryFactory queryFactory) {
this.databaseRepository = databaseRepository; this.databaseRepository = databaseRepository;
this.datasetRepository = datasetRepository; this.datasetRepository = datasetRepository;
this.entityManager = entityManager; this.entityManager = entityManager;
this.queryFactory = queryFactory;
} }
public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
@ -47,6 +54,12 @@ public class TagMigrationService {
Map<TagKey, UUID> savedTagIdsByName = new HashMap<>(); Map<TagKey, UUID> savedTagIdsByName = new HashMap<>();
HashSet<UUID> existingTagIds = new HashSet<>(); HashSet<UUID> existingTagIds = new HashSet<>();
List<TagEntity> existingTags = this.queryFactory.query(TagQuery.class).collectAs(new BaseFieldSet().ensure(ReferenceType._id).ensure(ReferenceType._name));
for (TagEntity tag : existingTags){
existingTagIds.add(tag.getId());
savedTagIdsByName.put(new TagKey(tag), tag.getId());
}
List<Dataset> items; List<Dataset> items;
do { do {
items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
@ -112,6 +125,13 @@ public class TagMigrationService {
hashCode = Objects.hash(this.owner, this.name); hashCode = Objects.hash(this.owner, this.name);
} }
public TagKey(TagEntity tag) {
this.name = tag.getLabel();
if (tag.getCreatedById() != null) this.owner = tag.getCreatedById();
else this.owner = null;
hashCode = Objects.hash(this.owner, this.name);
}
public UUID getOwner() { public UUID getOwner() {
return owner; return owner;
} }