MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration fixes

This commit is contained in:
Efstratios Giannopoulos 2024-05-13 14:58:32 +03:00
parent 6583c6d07a
commit a1cb8efe1d
3 changed files with 26 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -167,7 +167,7 @@ public class DatasetMigrationService {
private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){ private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList(); List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) { if (itemDescriptionTemplates.isEmpty()) {
logger.warn("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); if (!item.getStatus().equals(Dataset.Status.DELETED.getValue())) logger.warn("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) { if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) {
DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity(); DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity();
dmpDescriptionTemplateEntity.setId(UUID.randomUUID()); dmpDescriptionTemplateEntity.setId(UUID.randomUUID());
@ -176,7 +176,7 @@ public class DatasetMigrationService {
dmpDescriptionTemplateEntity.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now()); dmpDescriptionTemplateEntity.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now()); dmpDescriptionTemplateEntity.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setSectionId(sectionId); dmpDescriptionTemplateEntity.setSectionId(sectionId);
dmpDescriptionTemplateEntity.setIsActive(IsActive.Active); dmpDescriptionTemplateEntity.setIsActive(!item.getStatus().equals(Dataset.Status.DELETED.getValue()) ? IsActive.Active : IsActive.Inactive);
this.entityManager.persist(dmpDescriptionTemplateEntity); this.entityManager.persist(dmpDescriptionTemplateEntity);
itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity); itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity);
} else { } else {
@ -189,7 +189,8 @@ public class DatasetMigrationService {
private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap) { private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, org.opencdmp.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap) {
if (this.conventionService.isNullOrEmpty(item.getProperties())) return null; if (this.conventionService.isNullOrEmpty(item.getProperties())) return null;
JSONObject jObject = new JSONObject(item.getProperties()); JSONObject jObject = new JSONObject(item.getProperties());
Map<String, Object> properties = jObject.toMap(); Map<String, Object> properties = new HashMap<>();
for (Map.Entry<String, Object> entry : jObject.toMap().entrySet()) properties.put(entry.getKey().trim(), entry.getValue());
PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity(); PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity();
propertyDefinitionEntity.setFieldSets(new HashMap<>()); propertyDefinitionEntity.setFieldSets(new HashMap<>());
@ -225,7 +226,7 @@ public class DatasetMigrationService {
continue; continue;
} }
this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); this.addSimpleField(item, propertyDefinitionEntity, currentFieldSet, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap);
} }
} }
} }
@ -315,13 +316,13 @@ public class DatasetMigrationService {
if (ordinal == SimpleFieldSetOrdinal) throw new MyApplicationException("Fieldset has migration simple fieldset ordinal " + SimpleFieldSetOrdinal); if (ordinal == SimpleFieldSetOrdinal) throw new MyApplicationException("Fieldset has migration simple fieldset ordinal " + SimpleFieldSetOrdinal);
this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap); this.addMultipleField(item, propertyDefinitionFieldSetEntity, ordinal, currentField, properties.get(key), referenceTypeDefinitionEntityMap, referenceMap);
} }
} }
} }
private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){ private void addMultipleField(Dataset item, PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null); PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null);
if (propertyDefinitionFieldSetItemEntity == null){ if (propertyDefinitionFieldSetItemEntity == null){
@ -331,10 +332,10 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity); propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
} }
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap));
} }
private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){ private void addSimpleField(Dataset item, PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null); PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId().trim(), null);
if (propertyDefinitionFieldSetEntity == null) { if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity(); propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
@ -352,7 +353,7 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst(); propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
} }
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap)); propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId().trim(), this.buildField(item, currentField, propertyValue, referenceTypeDefinitionEntityMap, referenceMap));
} }
@ -377,9 +378,9 @@ public class DatasetMigrationService {
propertyDefinitionFieldSetItemEntity.setComment(comment); propertyDefinitionFieldSetItemEntity.setComment(comment);
} }
private FieldEntity buildField(Dataset item, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){ private FieldEntity buildField(Dataset item, org.opencdmp.commons.types.descriptiontemplate.FieldEntity currentField, Object propertyValue, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
FieldEntity fieldEntity = new FieldEntity(); FieldEntity fieldEntity = new FieldEntity();
String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null; String textValue = propertyValue != null ? propertyValue.toString() : null;
if (textValue == null || textValue.isEmpty()) return fieldEntity; if (textValue == null || textValue.isEmpty()) return fieldEntity;
switch (currentField.getData().getFieldType()){ switch (currentField.getData().getFieldType()){
case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> fieldEntity.setTextValue(textValue.trim()); case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> fieldEntity.setTextValue(textValue.trim());
@ -416,7 +417,7 @@ public class DatasetMigrationService {
if(!this.conventionService.isNullOrEmpty(textValue)) { if(!this.conventionService.isNullOrEmpty(textValue)) {
Map<String, String> valuesParsed = migrationTools.tryParseJsonAsObjectString(Map.class, textValue); Map<String, String> valuesParsed = migrationTools.tryParseJsonAsObjectString(Map.class, textValue);
if (valuesParsed == null) { if (valuesParsed == null) {
valuesParsed = (Map<String, String>) properties.get(currentField.getId()); valuesParsed = (Map<String, String>) propertyValue;
} }
if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue); if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue);
String id = valuesParsed.getOrDefault("id", null); String id = valuesParsed.getOrDefault("id", null);
@ -999,7 +1000,7 @@ public class DatasetMigrationService {
data.setSourceType(ReferenceSourceType.Internal); data.setSourceType(ReferenceSourceType.Internal);
data.setSource("Internal"); data.setSource("Internal");
if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId()); if (dataset.getCreator() != null) data.setCreatedById(dataset.getCreator().getId());
logger.warn("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " reference auto created as internal " + referenceTypeDataEntity.getReferenceTypeId() + " for value " + this.jsonHandlingService.toJsonSafe(item)); //logger.warn("Migrate Dataset " + dataset.getId() + " " + dataset.getLabel() + " reference auto created as internal " + referenceTypeDataEntity.getReferenceTypeId() + " for value " + this.jsonHandlingService.toJsonSafe(item));
} }
ReferenceEntity existingEntity = referenceMap.getOrDefault(new ReferenceKey(data), null); ReferenceEntity existingEntity = referenceMap.getOrDefault(new ReferenceKey(data), null);

13
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/StorageFileMigrationService.java
View File

@ -63,9 +63,8 @@ public class StorageFileMigrationService {
logger.debug("Migrate FileUpload " + page * PageSize + " of " + total); logger.debug("Migrate FileUpload " + page * PageSize + " of " + total);
for (FileUpload item : items) { for (FileUpload item : items) {
entityManager.detach(item); entityManager.detach(item);
// File file = new File(this.environment.getProperty("file.storage") + item.getId().toString()); File file = new File(this.environment.getProperty("file.migration-storage") + item.getId().toString());
// if (!file.exists()) file = new File(this.environment.getProperty("temp.temp") + item.getId().toString()); if (!file.exists()) throw new MyApplicationException("Storage file not exist " + item.getId().toString());
// if (!file.exists()) throw new MyApplicationException("Storage file not exist " + item.getId().toString());
StorageFileEntity data = new StorageFileEntity(); StorageFileEntity data = new StorageFileEntity();
data.setId(item.getId()); data.setId(item.getId());
@ -80,11 +79,11 @@ public class StorageFileMigrationService {
data.setPurgedAt(null); data.setPurgedAt(null);
if (item.getCreator() == null) data.setOwnerId(item.getCreator().getId()); if (item.getCreator() == null) data.setOwnerId(item.getCreator().getId());
// File destinationFile = new File(Paths.get(this.environment.getProperty("file.mainstorage"), data.getFileRef()).toString()); File destinationFile = new File(Paths.get(this.environment.getProperty("file.migrated-storage"), data.getFileRef()).toString());
// boolean fileCopied = FileCopyUtils.copy(file, destinationFile) > 0; boolean fileCopied = FileCopyUtils.copy(file, destinationFile) > 0;
// if (!fileCopied) throw new MyApplicationException("Storage file not copied " + data.getId().toString()); if (!fileCopied) throw new MyApplicationException("Storage file not copied " + data.getId().toString());
// //
// filesToDelete.add(file); filesToDelete.add(file);
this.entityManager.persist(data); this.entityManager.persist(data);
} }

10
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -113,12 +113,12 @@ public class MigrationController {
// this.dmpMigrationService.migrate(); // this.dmpMigrationService.migrate();
// this.dmpDatasetProfileMigrationService.migrate(); // this.dmpDatasetProfileMigrationService.migrate();
// this.dmpUserMigrationService.migrate(); // this.dmpUserMigrationService.migrate();
//
//Description
this.datasetMigrationService.migrate();
this.datasetReferenceMigrationService.migrateDatasetReferences();
this.tagMigrationService.migrate();
// //Description
// this.datasetMigrationService.migrate();
// this.datasetReferenceMigrationService.migrateDatasetReferences();
// this.tagMigrationService.migrate();
//
this.storageFileMigrationService.migrate(); this.storageFileMigrationService.migrate();
// throw new RuntimeException(""); // throw new RuntimeException("");