argos/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java

223 lines
14 KiB
Java
Raw Normal View History

2023-11-23 09:26:40 +01:00
package eu.old.eudat.migration;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.commons.JsonHandlingService;
2023-11-23 14:23:49 +01:00
import eu.eudat.commons.XmlHandlingService;
2023-11-23 09:26:40 +01:00
import eu.eudat.commons.enums.*;
2023-11-23 17:09:17 +01:00
import eu.eudat.commons.types.description.FieldEntity;
import eu.eudat.commons.types.description.PropertyDefinitionEntity;
2024-02-21 17:38:28 +01:00
import eu.eudat.commons.types.description.PropertyDefinitionFieldSetEntity;
import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
2023-11-23 14:23:49 +01:00
import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
import eu.eudat.commons.types.dmp.DmpContactEntity;
import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
import eu.eudat.commons.types.dmpblueprint.DefinitionEntity;
2024-02-21 17:38:28 +01:00
import eu.eudat.data.*;
2023-11-23 17:09:17 +01:00
import eu.eudat.model.Dmp;
import eu.eudat.model.DmpDescriptionTemplate;
2024-02-21 17:38:28 +01:00
import eu.eudat.query.DescriptionTemplateQuery;
2023-11-23 14:23:49 +01:00
import eu.eudat.query.DmpBlueprintQuery;
2023-11-23 17:09:17 +01:00
import eu.eudat.query.DmpDescriptionTemplateQuery;
import eu.eudat.query.DmpQuery;
2023-11-23 14:23:49 +01:00
import eu.old.eudat.data.dao.entities.DatasetDao;
import eu.old.eudat.data.entities.DMP;
import eu.old.eudat.data.entities.DMPDatasetProfile;
2023-11-23 09:26:40 +01:00
import eu.old.eudat.data.entities.Dataset;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import eu.old.eudat.models.data.datasetwizard.DatasetWizardModel;
2023-11-23 14:23:49 +01:00
import eu.old.eudat.models.data.dmp.DataManagementPlan;
2023-11-23 09:26:40 +01:00
import eu.old.eudat.queryable.QueryableList;
2023-11-23 14:23:49 +01:00
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
2023-11-23 17:09:17 +01:00
import gr.cite.tools.fieldset.BaseFieldSet;
2023-11-23 14:23:49 +01:00
import gr.cite.tools.logging.LoggerService;
2023-11-23 09:26:40 +01:00
import jakarta.persistence.EntityManager;
2023-11-23 14:23:49 +01:00
import jakarta.xml.bind.JAXBException;
2023-11-23 17:09:17 +01:00
import org.json.JSONObject;
2023-11-23 14:23:49 +01:00
import org.slf4j.LoggerFactory;
2023-11-23 09:26:40 +01:00
import org.springframework.stereotype.Service;
2023-11-23 14:23:49 +01:00
import org.xml.sax.SAXException;
2023-11-23 09:26:40 +01:00
2023-11-23 14:23:49 +01:00
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.time.Instant;
import java.util.*;
2023-11-23 17:09:17 +01:00
import java.util.stream.Collectors;
2023-11-23 09:26:40 +01:00
@Service
public class DatasetMigrationService {
2023-11-23 14:23:49 +01:00
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class));
2023-11-23 09:26:40 +01:00
private final DatabaseRepository databaseRepository;
private final JsonHandlingService jsonHandlingService;
2023-11-23 14:23:49 +01:00
private final QueryFactory queryFactory;
private final XmlHandlingService xmlHandlingService;
private static final int PageSize = 500;
private static final boolean TestMode = false;
2023-11-23 09:26:40 +01:00
private final EntityManager entityManager;
2023-11-23 14:23:49 +01:00
public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager) {
2023-11-23 09:26:40 +01:00
this.databaseRepository = databaseRepository;
this.jsonHandlingService = jsonHandlingService;
2023-11-23 14:23:49 +01:00
this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService;
2023-11-23 09:26:40 +01:00
this.entityManager = entityManager;
}
2023-11-23 14:23:49 +01:00
public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
DatasetDao datasetDao = databaseRepository.getDatasetDao();
long total = datasetDao.asQueryable().count();
logger.debug("Migrate Dataset Total : " + total);
int page = 0;
List<Dataset> items;
do {
2023-11-23 17:09:17 +01:00
items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
2023-11-23 14:23:49 +01:00
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dataset " + page * PageSize + " of " + total);
2023-11-23 17:09:17 +01:00
List<DmpBlueprintEntity> dmpBlueprints = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().map(x-> x.getDmp().getProfile().getId()).distinct().toList()).collect();
Map<UUID, DefinitionEntity> dmpBlueprintsMap = new HashMap<>();
for (DmpBlueprintEntity dmpBlueprint : dmpBlueprints) {
DefinitionEntity definitionEntity = this.xmlHandlingService.fromXml(DefinitionEntity.class, dmpBlueprint.getDefinition());
dmpBlueprintsMap.put(dmpBlueprint.getId(), definitionEntity);
}
2024-02-21 17:38:28 +01:00
List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities = this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collect();
List<DescriptionTemplateEntity> descriptionTemplates = this.queryFactory.query(DescriptionTemplateQuery.class).ids(items.stream().map(x-> x.getProfile().getId()).distinct().toList()).collect();
2023-11-23 17:09:17 +01:00
Map<UUID, UUID> creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId));;
2024-02-21 17:38:28 +01:00
Map<UUID, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity> descriptionTemplateDefinitionMap = new HashMap<>();
for (DescriptionTemplateEntity descriptionTemplateEntity : descriptionTemplates){
descriptionTemplateDefinitionMap.put(descriptionTemplateEntity.getId(), this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition()));
}
2023-11-23 14:23:49 +01:00
for (Dataset item : items) {
//entityManager.detach(item);
2023-11-23 17:09:17 +01:00
//DatasetWizardModel model = new DatasetWizardModel();
//model.fromDataModel(item);
2023-11-23 14:23:49 +01:00
2023-11-23 17:09:17 +01:00
DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
2023-11-29 15:56:25 +01:00
if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) {
2023-11-23 17:09:17 +01:00
logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}
UUID sectionId = definition.getSections().get(item.getDmpSectionIndex()).getId();
if (sectionId == null) {
logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}
2024-02-21 17:38:28 +01:00
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = this.getOrCreateDmpDescriptionTemplateEntity(item, sectionId, dmpDescriptionTemplateEntities);
2023-11-23 17:09:17 +01:00
if (itemDescriptionTemplates.size() > 1) {
logger.error("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
//TODO CLEAN Duplicates dmp, DescriptionTemplate, SectionId
}
2023-11-23 14:23:49 +01:00
DescriptionEntity data = new DescriptionEntity();
2023-11-23 17:09:17 +01:00
data.setId(item.getId());
data.setDescription(item.getDescription());
if (item.getCreator() != null) {
data.setCreatedById(item.getCreator().getId());
} else {
data.setCreatedById(creatorsByDmp.getOrDefault(item.getDmp().getId(), null));
}
data.setDmpId(item.getDmp().getId());
data.setLabel(item.getLabel());
2024-02-21 17:38:28 +01:00
data.setDmpDescriptionTemplateId(itemDescriptionTemplates.getFirst().getId());
2023-11-24 16:18:28 +01:00
data.setDescriptionTemplateId(item.getProfile().getId());
2023-11-23 17:09:17 +01:00
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
if (item.getFinalizedAt() != null)
data.setFinalizedAt(item.getFinalizedAt().toInstant());
if (item.getStatus() == 99) {
2023-11-23 14:23:49 +01:00
data.setIsActive(IsActive.Inactive);
data.setStatus(DescriptionStatus.Canceled);
} else {
data.setIsActive(IsActive.Active);
2023-11-23 17:09:17 +01:00
data.setStatus(DescriptionStatus.of(item.getStatus()));
}
2024-02-21 17:38:28 +01:00
eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null);
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity)));
2023-11-23 17:09:17 +01:00
2023-11-24 16:18:28 +01:00
if (data.getCreatedById() == null){
2023-11-23 17:09:17 +01:00
logger.error("Migration skipped creator not found " + item.getId());
throw new MyApplicationException("Migration skipped creator not found " + item.getId());
2023-11-23 14:23:49 +01:00
}
this.entityManager.persist(data);
this.entityManager.flush();
}
page++;
2023-11-23 09:26:40 +01:00
}
2023-11-23 14:23:49 +01:00
} while (items != null && !items.isEmpty() && !TestMode);
2023-11-23 09:26:40 +01:00
}
2024-02-21 17:38:28 +01:00
private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
if (dmpDescriptionTemplateEntities.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) {
DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity();
dmpDescriptionTemplateEntity.setId(UUID.randomUUID());
dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId());
dmpDescriptionTemplateEntity.setDmpId(item.getDmp().getId());
dmpDescriptionTemplateEntity.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setSectionId(sectionId);
dmpDescriptionTemplateEntity.setIsActive(IsActive.Active);
this.entityManager.persist(dmpDescriptionTemplateEntity);
itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity);
} else {
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
}
}
return itemDescriptionTemplates;
}
private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity) {
if (item.getProperties() != null) return null;
JSONObject jObject = new JSONObject(item.getProperties());
Map<String, Object> properties = jObject.toMap();
PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity();
List<FieldSetEntity> fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets();
// for (String key : properties.keySet()) {
// FieldSetEntity currentFieldSet = null;
// eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null;
// for (FieldSetEntity fieldSetEntity : fieldSetEntities){
// List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(key);
// if (!fieldEntities.isEmpty()){
// currentFieldSet = fieldSetEntity;
// currentField = fieldEntities.getFirst();
// }
// }
//
// if (currentFieldSet == null || currentField == null) throw new MyApplicationException("aaa");
//
// if (!propertyDefinitionEntity.getFieldSets().containsKey(currentFieldSet.getId())){
// PropertyDefinitionFieldSetEntity fieldSetEntity = new PropertyDefinitionFieldSetEntity();
// fieldSetEntity.setItems(new ArrayList<>());
// propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), fieldSetEntity);
// }
// propertyDefinitionEntity.getFieldSets().get(currentFieldSet.getId())
//
// FieldEntity fieldEntity = new FieldEntity();
// fieldEntity.setKey(key);
// fieldEntity.setValue(properties.get(key) != null ? properties.get(key).toString() : null);
// fieldEntities.add(fieldEntity);
// propertyDefinitionEntity.getFieldSets().put(key, fieldEntity);
// }
// propertyDefinitionEntity.setFields(fieldEntities);
return propertyDefinitionEntity;
}
2023-11-23 09:26:40 +01:00
}