diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/enums/FieldType.java b/dmp-backend/core/src/main/java/eu/eudat/commons/enums/FieldType.java index f6cb530d1..0cbac7bad 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/enums/FieldType.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/enums/FieldType.java @@ -68,8 +68,7 @@ public enum FieldType implements DatabaseEnum { public static boolean isTextType(FieldType fieldType){ return fieldType.equals(FieldType.FREE_TEXT) || fieldType.equals(FieldType.CHECK_BOX) || fieldType.equals(FieldType.TEXT_AREA) || fieldType.equals(FieldType.RICH_TEXT_AREA) || fieldType.equals(FieldType.UPLOAD) || fieldType.equals(FieldType.BOOLEAN_DECISION) || - fieldType.equals(FieldType.RADIO_BOX) || fieldType.equals(FieldType.CURRENCY) || fieldType.equals(FieldType.SELECT) || - fieldType.equals(FieldType.INTERNAL_ENTRIES_DMPS) || fieldType.equals(FieldType.INTERNAL_ENTRIES_DESCRIPTIONS); + fieldType.equals(FieldType.RADIO_BOX) || fieldType.equals(FieldType.CURRENCY) || fieldType.equals(FieldType.SELECT); } public static boolean isTextListType(FieldType fieldType){ diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java index 3f681241e..f81ddd2c6 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java @@ -1,32 +1,22 @@ package eu.old.eudat.migration; -import com.fasterxml.jackson.core.JsonProcessingException; import eu.eudat.commons.JsonHandlingService; import eu.eudat.commons.XmlHandlingService; -import eu.eudat.commons.enums.*; -import eu.eudat.commons.types.description.FieldEntity; -import eu.eudat.commons.types.description.PropertyDefinitionEntity; -import eu.eudat.commons.types.description.PropertyDefinitionFieldSetEntity; +import eu.eudat.commons.enums.DescriptionStatus; +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.types.description.*; import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity; -import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity; -import eu.eudat.commons.types.dmp.DmpContactEntity; -import eu.eudat.commons.types.dmp.DmpPropertiesEntity; import eu.eudat.commons.types.dmpblueprint.DefinitionEntity; +import eu.eudat.convention.ConventionService; import eu.eudat.data.*; import eu.eudat.model.Dmp; -import eu.eudat.model.DmpDescriptionTemplate; import eu.eudat.query.DescriptionTemplateQuery; import eu.eudat.query.DmpBlueprintQuery; import eu.eudat.query.DmpDescriptionTemplateQuery; import eu.eudat.query.DmpQuery; import eu.old.eudat.data.dao.entities.DatasetDao; -import eu.old.eudat.data.entities.DMP; -import eu.old.eudat.data.entities.DMPDatasetProfile; import eu.old.eudat.data.entities.Dataset; import eu.old.eudat.logic.services.operations.DatabaseRepository; -import eu.old.eudat.models.data.datasetwizard.DatasetWizardModel; -import eu.old.eudat.models.data.dmp.DataManagementPlan; -import eu.old.eudat.queryable.QueryableList; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.fieldset.BaseFieldSet; @@ -41,6 +31,9 @@ import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.time.Instant; +import java.time.LocalDate; +import java.time.ZoneOffset; +import java.time.format.DateTimeParseException; import java.util.*; import java.util.stream.Collectors; @@ -55,13 +48,15 @@ public class DatasetMigrationService { private static final int PageSize = 500; private static final boolean TestMode = false; private final EntityManager entityManager; + private final ConventionService conventionService; - public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager) { + public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService) { this.databaseRepository = databaseRepository; this.jsonHandlingService = jsonHandlingService; this.queryFactory = queryFactory; this.xmlHandlingService = xmlHandlingService; this.entityManager = entityManager; + this.conventionService = conventionService; } public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { @@ -92,11 +87,6 @@ public class DatasetMigrationService { descriptionTemplateDefinitionMap.put(descriptionTemplateEntity.getId(), this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition())); } for (Dataset item : items) { - //entityManager.detach(item); - - //DatasetWizardModel model = new DatasetWizardModel(); - //model.fromDataModel(item); - DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null); if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) { @@ -141,13 +131,16 @@ public class DatasetMigrationService { } eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null); - data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity))); + List existingTags = new ArrayList<>(); + data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, existingTags))); if (data.getCreatedById() == null){ logger.error("Migration skipped creator not found " + item.getId()); throw new MyApplicationException("Migration skipped creator not found " + item.getId()); } + + this.persistTags(existingTags, data); this.entityManager.persist(data); this.entityManager.flush(); @@ -156,13 +149,27 @@ public class DatasetMigrationService { page++; } } while (items != null && !items.isEmpty() && !TestMode); + throw new MyApplicationException(""); + } + + private void persistTags(List existingTags, DescriptionEntity data){ + for (Tag tag : existingTags) { + TagEntity tagEntity = new TagEntity(); + tagEntity.setId(UUID.fromString(tag.getId())); + tagEntity.setLabel(tag.getName()); + tagEntity.setCreatedAt(Instant.now()); + tagEntity.setUpdatedAt(Instant.now()); + tagEntity.setCreatedById(data.getCreatedById()); + tagEntity.setIsActive(IsActive.Active); + this.entityManager.persist(tagEntity); + } } private List getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List dmpDescriptionTemplateEntities){ List itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList(); if (itemDescriptionTemplates.isEmpty()) { logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); - if (dmpDescriptionTemplateEntities.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) { + if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) { DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity(); dmpDescriptionTemplateEntity.setId(UUID.randomUUID()); dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId()); @@ -180,43 +187,314 @@ public class DatasetMigrationService { return itemDescriptionTemplates; } - private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity) { - if (item.getProperties() != null) return null; + private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, List existingTags) { + if (this.conventionService.isNullOrEmpty(item.getProperties())) return null; JSONObject jObject = new JSONObject(item.getProperties()); Map properties = jObject.toMap(); PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity(); + propertyDefinitionEntity.setFieldSets(new HashMap<>()); + List fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets(); -// for (String key : properties.keySet()) { -// FieldSetEntity currentFieldSet = null; -// eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null; -// for (FieldSetEntity fieldSetEntity : fieldSetEntities){ -// List fieldEntities = fieldSetEntity.getFieldById(key); -// if (!fieldEntities.isEmpty()){ -// currentFieldSet = fieldSetEntity; -// currentField = fieldEntities.getFirst(); -// } -// } -// -// if (currentFieldSet == null || currentField == null) throw new MyApplicationException("aaa"); -// -// if (!propertyDefinitionEntity.getFieldSets().containsKey(currentFieldSet.getId())){ -// PropertyDefinitionFieldSetEntity fieldSetEntity = new PropertyDefinitionFieldSetEntity(); -// fieldSetEntity.setItems(new ArrayList<>()); -// propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), fieldSetEntity); -// } -// propertyDefinitionEntity.getFieldSets().get(currentFieldSet.getId()) -// -// FieldEntity fieldEntity = new FieldEntity(); -// fieldEntity.setKey(key); -// fieldEntity.setValue(properties.get(key) != null ? properties.get(key).toString() : null); -// fieldEntities.add(fieldEntity); -// propertyDefinitionEntity.getFieldSets().put(key, fieldEntity); -// } -// propertyDefinitionEntity.setFields(fieldEntities); - return propertyDefinitionEntity; + for (String key : properties.keySet()) { + if (key.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){ + if (properties.get(key) != null && this.conventionService.isNullOrEmpty(properties.get(key).toString())) { continue; } + FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null); + + if (currentFieldSet == null) { + logger.error("Field set " + key + " not found for comment " + properties.get(key).toString()); + continue; + } + + this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(key).toString()); + } else { + if (!key.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) { + FieldSetEntity currentFieldSet = null; + eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null; + for (FieldSetEntity fieldSetEntity : fieldSetEntities) { + List fieldEntities = fieldSetEntity.getFieldById(key.trim()); + if (!fieldEntities.isEmpty()) { + currentFieldSet = fieldSetEntity; + currentField = fieldEntities.getFirst(); + } + } + if (currentField == null) { + logger.error("Field set not found for field " + key + " " + properties.get(key).toString()); + continue; + } + + this.addSimpleField(propertyDefinitionEntity, currentFieldSet, currentField, properties, existingTags); + } + } + } + + Map groupOrdinalMapping = new HashMap<>(); + for (String key : properties.keySet()) { + if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) { + String newKey = key.trim().substring("multiple_".length()); + String[] keyParts = newKey.split("_", 4); + String fieldSetId = ""; + String groupId = ""; + int ordinal = 0; + String fieldId = ""; + try { + if (keyParts.length == 4) { + fieldSetId = keyParts[0].trim(); + groupId = keyParts[1].trim(); + ordinal = Integer.parseInt(keyParts[2].trim()); + fieldId = keyParts[3].trim(); + } else if (keyParts.length == 3) { + fieldSetId = keyParts[0].trim(); + groupId = keyParts[1].trim(); + fieldId = keyParts[2].trim(); + } else { + logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString()); + continue; + //throw new MyApplicationException("Invalid multiple key " + key); + } + } catch (Exception e){ + logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString()); + } + + + + eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = descriptionTemplateDefinitionEntity.getFieldById(fieldId).stream().findFirst().orElse(null); + if (currentField == null) { + logger.error("Field set not found for field " + key + " " + properties.get(key).toString()); + continue; + } + + PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(fieldSetId, null); + if (ordinal == 0) { + if (propertyDefinitionFieldSetEntity != null ) { + if (groupOrdinalMapping.containsKey(groupId + fieldSetId)){ + ordinal = groupOrdinalMapping.get(groupId + fieldSetId); + } else { + if (propertyDefinitionFieldSetEntity.getItems().stream().anyMatch(x-> x.getOrdinal() == 0 && !x.getFields().isEmpty())) { + ordinal = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> !x.getFields().isEmpty()).map(PropertyDefinitionFieldSetItemEntity::getOrdinal).max(Comparator.comparing(x -> x)).orElse(0) + 1; + int finalOrdinal = ordinal; + if (propertyDefinitionFieldSetEntity.getItems().stream().anyMatch(x -> x.getOrdinal() == finalOrdinal && !x.getFields().isEmpty())) + throw new MyApplicationException("Invalid multiple key " + key); + logger.error("Ordinal of group set to " + ordinal + " " + key); + } + } + } else { + propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); + propertyDefinitionFieldSetEntity.setItems(new ArrayList<>()); + } + } + if (!groupOrdinalMapping.containsKey(groupId + fieldSetId)){ + groupOrdinalMapping.put(groupId + fieldSetId, ordinal); + } else { + if (groupOrdinalMapping.get(groupId + fieldSetId) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key); + } + if (propertyDefinitionFieldSetEntity == null) throw new MyApplicationException("Invalid multiple key group " + key); + + this.addMultipleField(propertyDefinitionFieldSetEntity, ordinal, currentField, properties, existingTags); + } + } + return propertyDefinitionEntity; + } + + private void addMultipleField(PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, List existingTags){ + PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null); + + if (propertyDefinitionFieldSetItemEntity == null){ + propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity(); + propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>()); + propertyDefinitionFieldSetItemEntity.setOrdinal(ordinal); + propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); + } + + propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, existingTags)); + } + + private void addSimpleField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, List existingTags){ + PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null); + if (propertyDefinitionFieldSetEntity == null) { + propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); + propertyDefinitionFieldSetEntity.setItems(new ArrayList<>()); + propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity); + } + + PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null; + if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){ + propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity(); + propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>()); + propertyDefinitionFieldSetItemEntity.setOrdinal(0); + propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); + } else { + propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); + } + + propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, existingTags)); } + + private void addSimpleCommentField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, String comment){ + PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null); + if (propertyDefinitionFieldSetEntity == null) { + propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); + propertyDefinitionFieldSetEntity.setItems(new ArrayList<>()); + propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity); + } + + PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null; + if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){ + propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity(); + propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>()); + propertyDefinitionFieldSetItemEntity.setOrdinal(0); + propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); + } else { + propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); + } + + propertyDefinitionFieldSetItemEntity.setComment(comment); + } + + private FieldEntity buildField(eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map properties, List existingTags){ + FieldEntity fieldEntity = new FieldEntity(); + String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null; + if (textValue == null || textValue.isEmpty()) return fieldEntity; + switch (currentField.getData().getFieldType()){ + case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> fieldEntity.setTextValue(textValue.trim()); + case CHECK_BOX, BOOLEAN_DECISION -> fieldEntity.setTextValue(textValue.trim().toLowerCase(Locale.ROOT)); + case DATE_PICKER -> { + Instant instant = null; + if(!this.conventionService.isNullOrEmpty(textValue)) { + try { + instant = Instant.parse(textValue); + } catch (DateTimeParseException ex) { + instant = LocalDate.parse(textValue).atStartOfDay().toInstant(ZoneOffset.UTC); + } + } + fieldEntity.setDateValue(instant); + } + case SELECT -> { + if(!this.conventionService.isNullOrEmpty(textValue)) { + String[] valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, textValue); + if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, this.cleanAsObjectString(textValue)); + fieldEntity.setTextListValue(valuesParsed == null ? List.of(textValue) : Arrays.stream(valuesParsed).toList()); + } + } + case DATASET_IDENTIFIER, VALIDATION -> { + if(!this.conventionService.isNullOrEmpty(textValue)) { + ExternalIdentifierEntity externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, textValue); + if (externalIdentifierEntity == null) externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, this.cleanAsObjectString(textValue)); + if (externalIdentifierEntity == null) throw new MyApplicationException("Could not parse dataset External Identifier : " + textValue); + fieldEntity.setExternalIdentifier(externalIdentifierEntity); + } + } + case UPLOAD -> { + if(!this.conventionService.isNullOrEmpty(textValue)) { + Map valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, textValue); + if (valuesParsed == null) { + valuesParsed = (Map) properties.get(currentField.getId()); + } + if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue)); + if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue); + String id = valuesParsed.getOrDefault("id", null); + if (id == null) throw new MyApplicationException("Could not parse upload : " + textValue); + fieldEntity.setTextValue(UUID.fromString(id).toString()); + } + } + case CURRENCY -> { + if(!this.conventionService.isNullOrEmpty(textValue)) { + Currency currency = this.jsonHandlingService.fromJsonSafe(Currency.class, textValue); + if (currency == null) currency = this.jsonHandlingService.fromJsonSafe(Currency.class, this.cleanAsObjectString(textValue)); + //if (currency == null) throw new MyApplicationException("Could not parse Currency : " + textValue); + //TODO: {"name":"Euro","value":"EUR"} what we want to keep ? + + fieldEntity.setTextValue(currency == null ? textValue : currency.getName()); + } + } + case TAGS -> { + if(!this.conventionService.isNullOrEmpty(textValue)) { + Tag[] tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, textValue); + if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, this.cleanAsObjectString(textValue)); + if (tags == null) { + Tag tag = this.jsonHandlingService.fromJsonSafe(Tag.class, textValue); + if (tag == null) tag = this.jsonHandlingService.fromJsonSafe(Tag.class, this.cleanAsObjectString(textValue)); + if (tag != null) tags = List.of(tag).toArray(Tag[]::new); + } + if (tags == null) + logger.error("Could not parse tag : " + textValue); + else { + fieldEntity.setTextListValue(new ArrayList<>()); + for (Tag tag : tags){ + Tag existingTag = existingTags.stream().filter(x-> x.getName().equalsIgnoreCase(tag.getName())).findFirst().orElse(null); + if (existingTag == null){ + try { + UUID.fromString(tag.getId()); + } catch (IllegalArgumentException e){ + tag.setId(UUID.randomUUID().toString()); + } + existingTag = tag; + existingTags.add(tag); + } + fieldEntity.getTextListValue().add(existingTag.getId()); + } + } + //TODO: should create Description tag maybe should also check the persist + //TODO: migrate values maybe we should merge + } + } + + case INTERNAL_ENTRIES_DMPS -> fieldEntity.setTextValue(textValue.trim()); + case INTERNAL_ENTRIES_DESCRIPTIONS -> fieldEntity.setTextValue(textValue.trim()); + case REFERENCE_TYPES -> fieldEntity.setTextValue(textValue.trim()); + case EXTERNAL_DATASETS -> fieldEntity.setTextValue(textValue.trim()); + } + return fieldEntity; + } + + private String cleanAsObjectString(String value){ + value = value.trim().replace("\\", ""); + while (!this.conventionService.isNullOrEmpty(value) && value.startsWith("\"")) value = value.substring(1); + while (!this.conventionService.isNullOrEmpty(value) && value.endsWith("\"")) value = value.substring(0, value.length() - 1); + return value; + } + + public static class Currency{ + private String name; + private String value; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + } + public static class Tag{ + private String id; + private String name; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java index 95ae614ef..2ab756b4f 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java @@ -90,9 +90,13 @@ public class DmpMigrationService { Map> referenceTypeFieldsByDmpBlueprintId = new HashMap<>(); for (DmpBlueprintEntity dmpBlueprintEntity : dmpBlueprintEntities) { eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition = this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition()); - List referenceTypeFieldEntities = definition.getSections().stream().filter(x-> x.getFields() != null).map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x).collect(Collectors.toList()); + List referenceTypeFieldEntities = definition.getSections().stream().filter(x-> x.getFields() != null).map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x) + .filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.License) || x.getReferenceTypeId().equals(ReferenceTypeIds.Project) || x.getReferenceTypeId().equals(ReferenceTypeIds.Researcher) + || x.getReferenceTypeId().equals(ReferenceTypeIds.Grants) || x.getReferenceTypeId().equals(ReferenceTypeIds.Organizations) || x.getReferenceTypeId().equals(ReferenceTypeIds.Funder)) + .collect(Collectors.toList()); referenceTypeFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , referenceTypeFieldEntities); } + for (DMP item : items) { //entityManager.detach(item); @@ -212,7 +216,6 @@ public class DmpMigrationService { data.setIsActive(IsActive.Active); this.entityManager.persist(data); } - this.entityManager.flush(); } private void migrateProjects(DMP item, List referenceTypeFieldEntities){ @@ -237,7 +240,6 @@ public class DmpMigrationService { data.setIsActive(IsActive.Active); data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity))); this.entityManager.persist(data); - this.entityManager.flush(); } private void migrateResearchers(DMP item, List referenceTypeFieldEntities){ @@ -264,7 +266,6 @@ public class DmpMigrationService { data.setIsActive(IsActive.Active); this.entityManager.persist(data); } - this.entityManager.flush(); } private void migrateGrantAndFunder(DMP item, List referenceTypeFieldEntities){ @@ -306,7 +307,6 @@ public class DmpMigrationService { founder.setIsActive(IsActive.Active); this.entityManager.persist(founder); } - this.entityManager.flush(); } private Map migrateLicense(DMP item, DataManagementPlan model, Map licenseIdByName, List referenceTypeFieldEntities) throws JsonProcessingException { @@ -404,7 +404,6 @@ public class DmpMigrationService { entityManager.persist(dmpReferenceEntity); } } - this.entityManager.flush(); return licenseIdByName; } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/StorageFileMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/StorageFileMigrationService.java new file mode 100644 index 000000000..bdff209c7 --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/StorageFileMigrationService.java @@ -0,0 +1,105 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.XmlHandlingService; +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceFieldDataType; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.StorageType; +import eu.eudat.commons.types.reference.DefinitionEntity; +import eu.eudat.commons.types.reference.FieldEntity; +import eu.eudat.convention.ConventionService; +import eu.eudat.data.ReferenceEntity; +import eu.eudat.data.StorageFileEntity; +import eu.old.eudat.data.dao.entities.FileUploadDao; +import eu.old.eudat.data.dao.entities.ProjectDao; +import eu.old.eudat.data.entities.FileUpload; +import eu.old.eudat.data.entities.Project; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.exception.MyApplicationException; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.apache.commons.io.FilenameUtils; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.net.URLConnection; +import java.nio.file.Paths; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.UUID; +import org.springframework.core.env.Environment; +import org.springframework.util.FileCopyUtils; + +@Service +public class StorageFileMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(StorageFileMigrationService.class)); + + private static final int PageSize = 500; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + private final Environment environment; + public StorageFileMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager, Environment environment) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + this.environment = environment; + } + + public void migrate() throws IOException { + FileUploadDao fileUploadDao = databaseRepository.getFileUploadDao(); + long total = fileUploadDao.asQueryable().count(); + logger.debug("Migrate Files Total : " + total); + int page = 0; + + List filesToDelete = new ArrayList<>(); + + List items; + do { + items = fileUploadDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("createdAt"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate FileUpload " + page * PageSize + " of " + total); + for (FileUpload item : items) { + entityManager.detach(item); + File file = new File(this.environment.getProperty("file.storage") + item.getId().toString()); + if (!file.exists()) file = new File(this.environment.getProperty("temp.temp") + item.getId().toString()); + if (!file.exists()) throw new MyApplicationException("Storage file not exist " + item.getId().toString()); + + StorageFileEntity data = new StorageFileEntity(); + data.setId(item.getId()); + data.setFileRef(UUID.randomUUID().toString().replaceAll("-", "").toLowerCase(Locale.ROOT)); + data.setStorageType(StorageType.Main); + data.setName(FilenameUtils.removeExtension(item.getName())); + data.setExtension(FilenameUtils.getExtension(item.getName())); + data.setMimeType(item.getFileType()); + //data.setMimeType(URLConnection.guessContentTypeFromName(item.getName())); + data.setCreatedAt(item.getCreatedAt() == null ? Instant.now() : item.getCreatedAt().toInstant()); + data.setPurgeAt(null); + data.setPurgedAt(null); + if (item.getCreator() == null) data.setOwnerId(item.getCreator().getId()); + + File destinationFile = new File(Paths.get(this.environment.getProperty("file.mainstorage"), data.getFileRef()).toString()); + boolean fileCopied = FileCopyUtils.copy(file, destinationFile) > 0; + if (!fileCopied) throw new MyApplicationException("Storage file not copied " + data.getId().toString()); + + filesToDelete.add(file); + + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + throw new MyApplicationException(""); +// for (File file : filesToDelete){ +// try{ +// file.delete(); +// } catch (Exception e){ +// logger.error(e.getMessage()); +// } +// } + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java index d590ea300..2b411afb2 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java @@ -59,7 +59,6 @@ public class TagMigrationService { logger.debug("Migrate Dataset tags " + page * PageSize + " of " + total); criteria.setIds(items.stream().map(Dataset::getId).toList()); -// List elasticDatasets = this.datasetRepository.queryIds(criteria); List elasticDatasets = this.datasetRepository.findByIds(items.stream().map(x -> x.getId().toString()).toList()); for (Dataset item : items) { @@ -72,7 +71,7 @@ public class TagMigrationService { boolean tagAlreadyExists; if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) { for(Tag tag : elasticDataset.getTags()) { - tagAlreadyExists = savedTagNames.contains(tag.getName()); + tagAlreadyExists = savedTagNames.contains(tag.getName()); //TODO maybe should check owner to recreate savedTagNames.add(tag.getName()); if (!tagAlreadyExists) { TagEntity tagEntity = new TagEntity(); @@ -80,6 +79,7 @@ public class TagMigrationService { tagEntity.setLabel(tag.getName()); tagEntity.setCreatedAt(Instant.now()); tagEntity.setUpdatedAt(Instant.now()); + if (item.getCreator() != null) tagEntity.setCreatedById(item.getCreator().getId()); tagEntity.setIsActive(IsActive.Active); savedTagIdsByName.put(tag.getName(), tagEntity.getId()); this.entityManager.persist(tagEntity); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java index 5dd3d7067..9b704f9c0 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java @@ -43,6 +43,7 @@ public class MigrationController { private final DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService; private final DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService; private final DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService; + private final StorageFileMigrationService storageFileMigrationService; public MigrationController( DmpMigrationService dmpMigrationService, @@ -62,7 +63,7 @@ public class MigrationController { DmpUserMigrationService dmpUserMigrationService, UserMigrationService userMigrationService, DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService, - DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService, DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService) { + DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService, DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService, StorageFileMigrationService storageFileMigrationService) { this.dmpMigrationService = dmpMigrationService; this.datasetMigrationService = datasetMigrationService; this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService; @@ -83,6 +84,7 @@ public class MigrationController { this.dmpBlueprintXmlMigrationService = dmpBlueprintXmlMigrationService; this.descriptionTemplateXmlMigrationService = descriptionTemplateXmlMigrationService; this.descriptionTemplateXmlCleanInvalidReferenceTypesService = descriptionTemplateXmlCleanInvalidReferenceTypesService; + this.storageFileMigrationService = storageFileMigrationService; } @GetMapping("all") @@ -112,7 +114,9 @@ public class MigrationController { this.dmpUserMigrationService.migrate(); - this.referenceMigrationService.migrateDatasetReferences(); //TODO + //this.referenceMigrationService.migrateDatasetReferences(); //TODO + + this.storageFileMigrationService.migrate(); return true; } @@ -125,6 +129,14 @@ public class MigrationController { } + @GetMapping("storage-files") + @Transactional + public boolean migrateStorageFile() throws IOException { + this.storageFileMigrationService.migrate(); + return true; + + } + @GetMapping("description-template-xml") @Transactional public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {