MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration changes

This commit is contained in:
Efstratios Giannopoulos 2024-02-22 18:33:52 +02:00
parent 6ebbb885fb
commit a28323dcaf
6 changed files with 457 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
dmp-backend/core/src/main/java/eu/eudat/commons/enums/FieldType.java
View File

@ -68,8 +68,7 @@ public enum FieldType implements DatabaseEnum<String> {
public static boolean isTextType(FieldType fieldType){ public static boolean isTextType(FieldType fieldType){
return fieldType.equals(FieldType.FREE_TEXT) || fieldType.equals(FieldType.CHECK_BOX) || fieldType.equals(FieldType.TEXT_AREA) || return fieldType.equals(FieldType.FREE_TEXT) || fieldType.equals(FieldType.CHECK_BOX) || fieldType.equals(FieldType.TEXT_AREA) ||
fieldType.equals(FieldType.RICH_TEXT_AREA) || fieldType.equals(FieldType.UPLOAD) || fieldType.equals(FieldType.BOOLEAN_DECISION) || fieldType.equals(FieldType.RICH_TEXT_AREA) || fieldType.equals(FieldType.UPLOAD) || fieldType.equals(FieldType.BOOLEAN_DECISION) ||
fieldType.equals(FieldType.RADIO_BOX) || fieldType.equals(FieldType.CURRENCY) || fieldType.equals(FieldType.SELECT) || fieldType.equals(FieldType.RADIO_BOX) || fieldType.equals(FieldType.CURRENCY) || fieldType.equals(FieldType.SELECT);
fieldType.equals(FieldType.INTERNAL_ENTRIES_DMPS) || fieldType.equals(FieldType.INTERNAL_ENTRIES_DESCRIPTIONS);
} }
public static boolean isTextListType(FieldType fieldType){ public static boolean isTextListType(FieldType fieldType){

382
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -1,32 +1,22 @@
package eu.old.eudat.migration; package eu.old.eudat.migration;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.commons.JsonHandlingService; import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.*; import eu.eudat.commons.enums.DescriptionStatus;
import eu.eudat.commons.types.description.FieldEntity; import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.types.description.PropertyDefinitionEntity; import eu.eudat.commons.types.description.*;
import eu.eudat.commons.types.description.PropertyDefinitionFieldSetEntity;
import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity; import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
import eu.eudat.commons.types.dmp.DmpContactEntity;
import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
import eu.eudat.commons.types.dmpblueprint.DefinitionEntity; import eu.eudat.commons.types.dmpblueprint.DefinitionEntity;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.*; import eu.eudat.data.*;
import eu.eudat.model.Dmp; import eu.eudat.model.Dmp;
import eu.eudat.model.DmpDescriptionTemplate;
import eu.eudat.query.DescriptionTemplateQuery; import eu.eudat.query.DescriptionTemplateQuery;
import eu.eudat.query.DmpBlueprintQuery; import eu.eudat.query.DmpBlueprintQuery;
import eu.eudat.query.DmpDescriptionTemplateQuery; import eu.eudat.query.DmpDescriptionTemplateQuery;
import eu.eudat.query.DmpQuery; import eu.eudat.query.DmpQuery;
import eu.old.eudat.data.dao.entities.DatasetDao; import eu.old.eudat.data.dao.entities.DatasetDao;
import eu.old.eudat.data.entities.DMP;
import eu.old.eudat.data.entities.DMPDatasetProfile;
import eu.old.eudat.data.entities.Dataset; import eu.old.eudat.data.entities.Dataset;
import eu.old.eudat.logic.services.operations.DatabaseRepository; import eu.old.eudat.logic.services.operations.DatabaseRepository;
import eu.old.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.old.eudat.models.data.dmp.DataManagementPlan;
import eu.old.eudat.queryable.QueryableList;
import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.fieldset.BaseFieldSet; import gr.cite.tools.fieldset.BaseFieldSet;
@ -41,6 +31,9 @@ import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException; import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException; import java.io.IOException;
import java.time.Instant; import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
import java.util.*; import java.util.*;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -55,13 +48,15 @@ public class DatasetMigrationService {
private static final int PageSize = 500; private static final int PageSize = 500;
private static final boolean TestMode = false; private static final boolean TestMode = false;
private final EntityManager entityManager; private final EntityManager entityManager;
private final ConventionService conventionService;
public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager) { public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService) {
this.databaseRepository = databaseRepository; this.databaseRepository = databaseRepository;
this.jsonHandlingService = jsonHandlingService; this.jsonHandlingService = jsonHandlingService;
this.queryFactory = queryFactory; this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService; this.xmlHandlingService = xmlHandlingService;
this.entityManager = entityManager; this.entityManager = entityManager;
this.conventionService = conventionService;
} }
public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
@ -92,11 +87,6 @@ public class DatasetMigrationService {
descriptionTemplateDefinitionMap.put(descriptionTemplateEntity.getId(), this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition())); descriptionTemplateDefinitionMap.put(descriptionTemplateEntity.getId(), this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition()));
} }
for (Dataset item : items) { for (Dataset item : items) {
//entityManager.detach(item);
//DatasetWizardModel model = new DatasetWizardModel();
//model.fromDataModel(item);
DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null); DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) { if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) {
@ -141,14 +131,17 @@ public class DatasetMigrationService {
} }
eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null); eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null);
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity))); List<Tag> existingTags = new ArrayList<>();
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, existingTags)));
if (data.getCreatedById() == null){ if (data.getCreatedById() == null){
logger.error("Migration skipped creator not found " + item.getId()); logger.error("Migration skipped creator not found " + item.getId());
throw new MyApplicationException("Migration skipped creator not found " + item.getId()); throw new MyApplicationException("Migration skipped creator not found " + item.getId());
} }
this.persistTags(existingTags, data);
this.entityManager.persist(data); this.entityManager.persist(data);
this.entityManager.flush(); this.entityManager.flush();
} }
@ -156,13 +149,27 @@ public class DatasetMigrationService {
page++; page++;
} }
} while (items != null && !items.isEmpty() && !TestMode); } while (items != null && !items.isEmpty() && !TestMode);
throw new MyApplicationException("");
}
private void persistTags(List<Tag> existingTags, DescriptionEntity data){
for (Tag tag : existingTags) {
TagEntity tagEntity = new TagEntity();
tagEntity.setId(UUID.fromString(tag.getId()));
tagEntity.setLabel(tag.getName());
tagEntity.setCreatedAt(Instant.now());
tagEntity.setUpdatedAt(Instant.now());
tagEntity.setCreatedById(data.getCreatedById());
tagEntity.setIsActive(IsActive.Active);
this.entityManager.persist(tagEntity);
}
} }
private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){ private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList(); List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) { if (itemDescriptionTemplates.isEmpty()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
if (dmpDescriptionTemplateEntities.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) { if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) {
DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity(); DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity();
dmpDescriptionTemplateEntity.setId(UUID.randomUUID()); dmpDescriptionTemplateEntity.setId(UUID.randomUUID());
dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId()); dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId());
@ -180,43 +187,314 @@ public class DatasetMigrationService {
return itemDescriptionTemplates; return itemDescriptionTemplates;
} }
private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity) { private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, List<Tag> existingTags) {
if (item.getProperties() != null) return null; if (this.conventionService.isNullOrEmpty(item.getProperties())) return null;
JSONObject jObject = new JSONObject(item.getProperties()); JSONObject jObject = new JSONObject(item.getProperties());
Map<String, Object> properties = jObject.toMap(); Map<String, Object> properties = jObject.toMap();
PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity(); PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity();
propertyDefinitionEntity.setFieldSets(new HashMap<>());
List<FieldSetEntity> fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets(); List<FieldSetEntity> fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets();
// for (String key : properties.keySet()) { for (String key : properties.keySet()) {
// FieldSetEntity currentFieldSet = null; if (key.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){
// eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null; if (properties.get(key) != null && this.conventionService.isNullOrEmpty(properties.get(key).toString())) { continue; }
// for (FieldSetEntity fieldSetEntity : fieldSetEntities){ FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null);
// List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(key);
// if (!fieldEntities.isEmpty()){
// currentFieldSet = fieldSetEntity;
// currentField = fieldEntities.getFirst();
// }
// }
//
// if (currentFieldSet == null || currentField == null) throw new MyApplicationException("aaa");
//
// if (!propertyDefinitionEntity.getFieldSets().containsKey(currentFieldSet.getId())){
// PropertyDefinitionFieldSetEntity fieldSetEntity = new PropertyDefinitionFieldSetEntity();
// fieldSetEntity.setItems(new ArrayList<>());
// propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), fieldSetEntity);
// }
// propertyDefinitionEntity.getFieldSets().get(currentFieldSet.getId())
//
// FieldEntity fieldEntity = new FieldEntity();
// fieldEntity.setKey(key);
// fieldEntity.setValue(properties.get(key) != null ? properties.get(key).toString() : null);
// fieldEntities.add(fieldEntity);
// propertyDefinitionEntity.getFieldSets().put(key, fieldEntity);
// }
// propertyDefinitionEntity.setFields(fieldEntities);
return propertyDefinitionEntity;
if (currentFieldSet == null) {
logger.error("Field set " + key + " not found for comment " + properties.get(key).toString());
continue;
} }
this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(key).toString());
} else {
if (!key.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) {
FieldSetEntity currentFieldSet = null;
eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null;
for (FieldSetEntity fieldSetEntity : fieldSetEntities) {
List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(key.trim());
if (!fieldEntities.isEmpty()) {
currentFieldSet = fieldSetEntity;
currentField = fieldEntities.getFirst();
}
}
if (currentField == null) {
logger.error("Field set not found for field " + key + " " + properties.get(key).toString());
continue;
}
this.addSimpleField(propertyDefinitionEntity, currentFieldSet, currentField, properties, existingTags);
}
}
}
Map<String, Integer> groupOrdinalMapping = new HashMap<>();
for (String key : properties.keySet()) {
if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) {
String newKey = key.trim().substring("multiple_".length());
String[] keyParts = newKey.split("_", 4);
String fieldSetId = "";
String groupId = "";
int ordinal = 0;
String fieldId = "";
try {
if (keyParts.length == 4) {
fieldSetId = keyParts[0].trim();
groupId = keyParts[1].trim();
ordinal = Integer.parseInt(keyParts[2].trim());
fieldId = keyParts[3].trim();
} else if (keyParts.length == 3) {
fieldSetId = keyParts[0].trim();
groupId = keyParts[1].trim();
fieldId = keyParts[2].trim();
} else {
logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString());
continue;
//throw new MyApplicationException("Invalid multiple key " + key);
}
} catch (Exception e){
logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString());
}
eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = descriptionTemplateDefinitionEntity.getFieldById(fieldId).stream().findFirst().orElse(null);
if (currentField == null) {
logger.error("Field set not found for field " + key + " " + properties.get(key).toString());
continue;
}
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(fieldSetId, null);
if (ordinal == 0) {
if (propertyDefinitionFieldSetEntity != null ) {
if (groupOrdinalMapping.containsKey(groupId + fieldSetId)){
ordinal = groupOrdinalMapping.get(groupId + fieldSetId);
} else {
if (propertyDefinitionFieldSetEntity.getItems().stream().anyMatch(x-> x.getOrdinal() == 0 && !x.getFields().isEmpty())) {
ordinal = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> !x.getFields().isEmpty()).map(PropertyDefinitionFieldSetItemEntity::getOrdinal).max(Comparator.comparing(x -> x)).orElse(0) + 1;
int finalOrdinal = ordinal;
if (propertyDefinitionFieldSetEntity.getItems().stream().anyMatch(x -> x.getOrdinal() == finalOrdinal && !x.getFields().isEmpty()))
throw new MyApplicationException("Invalid multiple key " + key);
logger.error("Ordinal of group set to " + ordinal + " " + key);
}
}
} else {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
}
}
if (!groupOrdinalMapping.containsKey(groupId + fieldSetId)){
groupOrdinalMapping.put(groupId + fieldSetId, ordinal);
} else {
if (groupOrdinalMapping.get(groupId + fieldSetId) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key);
}
if (propertyDefinitionFieldSetEntity == null) throw new MyApplicationException("Invalid multiple key group " + key);
this.addMultipleField(propertyDefinitionFieldSetEntity, ordinal, currentField, properties, existingTags);
}
}
return propertyDefinitionEntity;
}
private void addMultipleField(PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null);
if (propertyDefinitionFieldSetItemEntity == null){
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
propertyDefinitionFieldSetItemEntity.setOrdinal(ordinal);
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
}
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, existingTags));
}
private void addSimpleField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity);
}
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null;
if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
propertyDefinitionFieldSetItemEntity.setOrdinal(0);
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
} else {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
}
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, existingTags));
}
private void addSimpleCommentField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, String comment){
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity);
}
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null;
if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
propertyDefinitionFieldSetItemEntity.setOrdinal(0);
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
} else {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
}
propertyDefinitionFieldSetItemEntity.setComment(comment);
}
private FieldEntity buildField(eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, List<Tag> existingTags){
FieldEntity fieldEntity = new FieldEntity();
String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null;
if (textValue == null || textValue.isEmpty()) return fieldEntity;
switch (currentField.getData().getFieldType()){
case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> fieldEntity.setTextValue(textValue.trim());
case CHECK_BOX, BOOLEAN_DECISION -> fieldEntity.setTextValue(textValue.trim().toLowerCase(Locale.ROOT));
case DATE_PICKER -> {
Instant instant = null;
if(!this.conventionService.isNullOrEmpty(textValue)) {
try {
instant = Instant.parse(textValue);
} catch (DateTimeParseException ex) {
instant = LocalDate.parse(textValue).atStartOfDay().toInstant(ZoneOffset.UTC);
}
}
fieldEntity.setDateValue(instant);
}
case SELECT -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
String[] valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, textValue);
if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, this.cleanAsObjectString(textValue));
fieldEntity.setTextListValue(valuesParsed == null ? List.of(textValue) : Arrays.stream(valuesParsed).toList());
}
}
case DATASET_IDENTIFIER, VALIDATION -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
ExternalIdentifierEntity externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, textValue);
if (externalIdentifierEntity == null) externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, this.cleanAsObjectString(textValue));
if (externalIdentifierEntity == null) throw new MyApplicationException("Could not parse dataset External Identifier : " + textValue);
fieldEntity.setExternalIdentifier(externalIdentifierEntity);
}
}
case UPLOAD -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Map<String, String> valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, textValue);
if (valuesParsed == null) {
valuesParsed = (Map<String, String>) properties.get(currentField.getId());
}
if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue));
if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue);
String id = valuesParsed.getOrDefault("id", null);
if (id == null) throw new MyApplicationException("Could not parse upload : " + textValue);
fieldEntity.setTextValue(UUID.fromString(id).toString());
}
}
case CURRENCY -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Currency currency = this.jsonHandlingService.fromJsonSafe(Currency.class, textValue);
if (currency == null) currency = this.jsonHandlingService.fromJsonSafe(Currency.class, this.cleanAsObjectString(textValue));
//if (currency == null) throw new MyApplicationException("Could not parse Currency : " + textValue);
//TODO: {"name":"Euro","value":"EUR"} what we want to keep ?
fieldEntity.setTextValue(currency == null ? textValue : currency.getName());
}
}
case TAGS -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Tag[] tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, textValue);
if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, this.cleanAsObjectString(textValue));
if (tags == null) {
Tag tag = this.jsonHandlingService.fromJsonSafe(Tag.class, textValue);
if (tag == null) tag = this.jsonHandlingService.fromJsonSafe(Tag.class, this.cleanAsObjectString(textValue));
if (tag != null) tags = List.of(tag).toArray(Tag[]::new);
}
if (tags == null)
logger.error("Could not parse tag : " + textValue);
else {
fieldEntity.setTextListValue(new ArrayList<>());
for (Tag tag : tags){
Tag existingTag = existingTags.stream().filter(x-> x.getName().equalsIgnoreCase(tag.getName())).findFirst().orElse(null);
if (existingTag == null){
try {
UUID.fromString(tag.getId());
} catch (IllegalArgumentException e){
tag.setId(UUID.randomUUID().toString());
}
existingTag = tag;
existingTags.add(tag);
}
fieldEntity.getTextListValue().add(existingTag.getId());
}
}
//TODO: should create Description tag maybe should also check the persist
//TODO: migrate values maybe we should merge
}
}
case INTERNAL_ENTRIES_DMPS -> fieldEntity.setTextValue(textValue.trim());
case INTERNAL_ENTRIES_DESCRIPTIONS -> fieldEntity.setTextValue(textValue.trim());
case REFERENCE_TYPES -> fieldEntity.setTextValue(textValue.trim());
case EXTERNAL_DATASETS -> fieldEntity.setTextValue(textValue.trim());
}
return fieldEntity;
}
private String cleanAsObjectString(String value){
value = value.trim().replace("\\", "");
while (!this.conventionService.isNullOrEmpty(value) && value.startsWith("\"")) value = value.substring(1);
while (!this.conventionService.isNullOrEmpty(value) && value.endsWith("\"")) value = value.substring(0, value.length() - 1);
return value;
}
public static class Currency{
private String name;
private String value;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
public void setValue(String value) {
this.value = value;
}
}
public static class Tag{
private String id;
private String name;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
} }

11
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java
View File

@ -90,9 +90,13 @@ public class DmpMigrationService {
Map<UUID, List<ReferenceTypeFieldEntity>> referenceTypeFieldsByDmpBlueprintId = new HashMap<>(); Map<UUID, List<ReferenceTypeFieldEntity>> referenceTypeFieldsByDmpBlueprintId = new HashMap<>();
for (DmpBlueprintEntity dmpBlueprintEntity : dmpBlueprintEntities) { for (DmpBlueprintEntity dmpBlueprintEntity : dmpBlueprintEntities) {
eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition = this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition()); eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition = this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition());
List<ReferenceTypeFieldEntity> referenceTypeFieldEntities = definition.getSections().stream().filter(x-> x.getFields() != null).map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x).collect(Collectors.toList()); List<ReferenceTypeFieldEntity> referenceTypeFieldEntities = definition.getSections().stream().filter(x-> x.getFields() != null).map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x)
.filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.License) || x.getReferenceTypeId().equals(ReferenceTypeIds.Project) || x.getReferenceTypeId().equals(ReferenceTypeIds.Researcher)
|| x.getReferenceTypeId().equals(ReferenceTypeIds.Grants) || x.getReferenceTypeId().equals(ReferenceTypeIds.Organizations) || x.getReferenceTypeId().equals(ReferenceTypeIds.Funder))
.collect(Collectors.toList());
referenceTypeFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , referenceTypeFieldEntities); referenceTypeFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , referenceTypeFieldEntities);
} }
for (DMP item : items) { for (DMP item : items) {
//entityManager.detach(item); //entityManager.detach(item);
@ -212,7 +216,6 @@ public class DmpMigrationService {
data.setIsActive(IsActive.Active); data.setIsActive(IsActive.Active);
this.entityManager.persist(data); this.entityManager.persist(data);
} }
this.entityManager.flush();
} }
private void migrateProjects(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){ private void migrateProjects(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
@ -237,7 +240,6 @@ public class DmpMigrationService {
data.setIsActive(IsActive.Active); data.setIsActive(IsActive.Active);
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity))); data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
this.entityManager.persist(data); this.entityManager.persist(data);
this.entityManager.flush();
} }
private void migrateResearchers(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){ private void migrateResearchers(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
@ -264,7 +266,6 @@ public class DmpMigrationService {
data.setIsActive(IsActive.Active); data.setIsActive(IsActive.Active);
this.entityManager.persist(data); this.entityManager.persist(data);
} }
this.entityManager.flush();
} }
private void migrateGrantAndFunder(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){ private void migrateGrantAndFunder(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
@ -306,7 +307,6 @@ public class DmpMigrationService {
founder.setIsActive(IsActive.Active); founder.setIsActive(IsActive.Active);
this.entityManager.persist(founder); this.entityManager.persist(founder);
} }
this.entityManager.flush();
} }
private Map<String, UUID> migrateLicense(DMP item, DataManagementPlan model, Map<String, UUID> licenseIdByName, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities) throws JsonProcessingException { private Map<String, UUID> migrateLicense(DMP item, DataManagementPlan model, Map<String, UUID> licenseIdByName, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities) throws JsonProcessingException {
@ -404,7 +404,6 @@ public class DmpMigrationService {
entityManager.persist(dmpReferenceEntity); entityManager.persist(dmpReferenceEntity);
} }
} }
this.entityManager.flush();
return licenseIdByName; return licenseIdByName;
} }

105
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/StorageFileMigrationService.java Normal file
View File

@ -0,0 +1,105 @@
package eu.old.eudat.migration;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.StorageType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.StorageFileEntity;
import eu.old.eudat.data.dao.entities.FileUploadDao;
import eu.old.eudat.data.dao.entities.ProjectDao;
import eu.old.eudat.data.entities.FileUpload;
import eu.old.eudat.data.entities.Project;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import org.apache.commons.io.FilenameUtils;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.io.File;
import java.io.IOException;
import java.net.URLConnection;
import java.nio.file.Paths;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.UUID;
import org.springframework.core.env.Environment;
import org.springframework.util.FileCopyUtils;
@Service
public class StorageFileMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(StorageFileMigrationService.class));
private static final int PageSize = 500;
private final DatabaseRepository databaseRepository;
private final EntityManager entityManager;
private final Environment environment;
public StorageFileMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager, Environment environment) {
this.databaseRepository = databaseRepository;
this.entityManager = entityManager;
this.environment = environment;
}
public void migrate() throws IOException {
FileUploadDao fileUploadDao = databaseRepository.getFileUploadDao();
long total = fileUploadDao.asQueryable().count();
logger.debug("Migrate Files Total : " + total);
int page = 0;
List<File> filesToDelete = new ArrayList<>();
List<FileUpload> items;
do {
items = fileUploadDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("createdAt"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate FileUpload " + page * PageSize + " of " + total);
for (FileUpload item : items) {
entityManager.detach(item);
File file = new File(this.environment.getProperty("file.storage") + item.getId().toString());
if (!file.exists()) file = new File(this.environment.getProperty("temp.temp") + item.getId().toString());
if (!file.exists()) throw new MyApplicationException("Storage file not exist " + item.getId().toString());
StorageFileEntity data = new StorageFileEntity();
data.setId(item.getId());
data.setFileRef(UUID.randomUUID().toString().replaceAll("-", "").toLowerCase(Locale.ROOT));
data.setStorageType(StorageType.Main);
data.setName(FilenameUtils.removeExtension(item.getName()));
data.setExtension(FilenameUtils.getExtension(item.getName()));
data.setMimeType(item.getFileType());
//data.setMimeType(URLConnection.guessContentTypeFromName(item.getName()));
data.setCreatedAt(item.getCreatedAt() == null ? Instant.now() : item.getCreatedAt().toInstant());
data.setPurgeAt(null);
data.setPurgedAt(null);
if (item.getCreator() == null) data.setOwnerId(item.getCreator().getId());
File destinationFile = new File(Paths.get(this.environment.getProperty("file.mainstorage"), data.getFileRef()).toString());
boolean fileCopied = FileCopyUtils.copy(file, destinationFile) > 0;
if (!fileCopied) throw new MyApplicationException("Storage file not copied " + data.getId().toString());
filesToDelete.add(file);
this.entityManager.persist(data);
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
throw new MyApplicationException("");
// for (File file : filesToDelete){
// try{
// file.delete();
// } catch (Exception e){
// logger.error(e.getMessage());
// }
// }
}
}

4
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/TagMigrationService.java
View File

@ -59,7 +59,6 @@ public class TagMigrationService {
logger.debug("Migrate Dataset tags " + page * PageSize + " of " + total); logger.debug("Migrate Dataset tags " + page * PageSize + " of " + total);
criteria.setIds(items.stream().map(Dataset::getId).toList()); criteria.setIds(items.stream().map(Dataset::getId).toList());
// List<eu.old.eudat.elastic.entities.Dataset> elasticDatasets = this.datasetRepository.queryIds(criteria);
List<eu.old.eudat.elastic.entities.Dataset> elasticDatasets = this.datasetRepository.findByIds(items.stream().map(x -> x.getId().toString()).toList()); List<eu.old.eudat.elastic.entities.Dataset> elasticDatasets = this.datasetRepository.findByIds(items.stream().map(x -> x.getId().toString()).toList());
for (Dataset item : items) { for (Dataset item : items) {
@ -72,7 +71,7 @@ public class TagMigrationService {
boolean tagAlreadyExists; boolean tagAlreadyExists;
if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) { if (elasticDataset.getTags() != null && !elasticDataset.getTags().isEmpty()) {
for(Tag tag : elasticDataset.getTags()) { for(Tag tag : elasticDataset.getTags()) {
tagAlreadyExists = savedTagNames.contains(tag.getName()); tagAlreadyExists = savedTagNames.contains(tag.getName()); //TODO maybe should check owner to recreate
savedTagNames.add(tag.getName()); savedTagNames.add(tag.getName());
if (!tagAlreadyExists) { if (!tagAlreadyExists) {
TagEntity tagEntity = new TagEntity(); TagEntity tagEntity = new TagEntity();
@ -80,6 +79,7 @@ public class TagMigrationService {
tagEntity.setLabel(tag.getName()); tagEntity.setLabel(tag.getName());
tagEntity.setCreatedAt(Instant.now()); tagEntity.setCreatedAt(Instant.now());
tagEntity.setUpdatedAt(Instant.now()); tagEntity.setUpdatedAt(Instant.now());
if (item.getCreator() != null) tagEntity.setCreatedById(item.getCreator().getId());
tagEntity.setIsActive(IsActive.Active); tagEntity.setIsActive(IsActive.Active);
savedTagIdsByName.put(tag.getName(), tagEntity.getId()); savedTagIdsByName.put(tag.getName(), tagEntity.getId());
this.entityManager.persist(tagEntity); this.entityManager.persist(tagEntity);

16
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -43,6 +43,7 @@ public class MigrationController {
private final DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService; private final DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService;
private final DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService; private final DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService;
private final DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService; private final DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService;
private final StorageFileMigrationService storageFileMigrationService;
public MigrationController( public MigrationController(
DmpMigrationService dmpMigrationService, DmpMigrationService dmpMigrationService,
@ -62,7 +63,7 @@ public class MigrationController {
DmpUserMigrationService dmpUserMigrationService, DmpUserMigrationService dmpUserMigrationService,
UserMigrationService userMigrationService, UserMigrationService userMigrationService,
DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService, DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService,
DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService, DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService) { DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService, DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService, StorageFileMigrationService storageFileMigrationService) {
this.dmpMigrationService = dmpMigrationService; this.dmpMigrationService = dmpMigrationService;
this.datasetMigrationService = datasetMigrationService; this.datasetMigrationService = datasetMigrationService;
this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService; this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
@ -83,6 +84,7 @@ public class MigrationController {
this.dmpBlueprintXmlMigrationService = dmpBlueprintXmlMigrationService; this.dmpBlueprintXmlMigrationService = dmpBlueprintXmlMigrationService;
this.descriptionTemplateXmlMigrationService = descriptionTemplateXmlMigrationService; this.descriptionTemplateXmlMigrationService = descriptionTemplateXmlMigrationService;
this.descriptionTemplateXmlCleanInvalidReferenceTypesService = descriptionTemplateXmlCleanInvalidReferenceTypesService; this.descriptionTemplateXmlCleanInvalidReferenceTypesService = descriptionTemplateXmlCleanInvalidReferenceTypesService;
this.storageFileMigrationService = storageFileMigrationService;
} }
@GetMapping("all") @GetMapping("all")
@ -112,7 +114,9 @@ public class MigrationController {
this.dmpUserMigrationService.migrate(); this.dmpUserMigrationService.migrate();
this.referenceMigrationService.migrateDatasetReferences(); //TODO //this.referenceMigrationService.migrateDatasetReferences(); //TODO
this.storageFileMigrationService.migrate();
return true; return true;
} }
@ -125,6 +129,14 @@ public class MigrationController {
} }
@GetMapping("storage-files")
@Transactional
public boolean migrateStorageFile() throws IOException {
this.storageFileMigrationService.migrate();
return true;
}
@GetMapping("description-template-xml") @GetMapping("description-template-xml")
@Transactional @Transactional
public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException { public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {