package eu.old.eudat.migration; import com.fasterxml.jackson.core.JsonProcessingException; import eu.eudat.commons.JsonHandlingService; import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.enums.*; import eu.eudat.commons.types.description.FieldEntity; import eu.eudat.commons.types.description.PropertyDefinitionEntity; import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity; import eu.eudat.commons.types.dmp.DmpContactEntity; import eu.eudat.commons.types.dmp.DmpPropertiesEntity; import eu.eudat.commons.types.dmpblueprint.DefinitionEntity; import eu.eudat.data.DescriptionEntity; import eu.eudat.data.DmpBlueprintEntity; import eu.eudat.data.DmpDescriptionTemplateEntity; import eu.eudat.data.DmpEntity; import eu.eudat.model.Dmp; import eu.eudat.model.DmpDescriptionTemplate; import eu.eudat.query.DmpBlueprintQuery; import eu.eudat.query.DmpDescriptionTemplateQuery; import eu.eudat.query.DmpQuery; import eu.old.eudat.data.dao.entities.DatasetDao; import eu.old.eudat.data.entities.DMP; import eu.old.eudat.data.entities.DMPDatasetProfile; import eu.old.eudat.data.entities.Dataset; import eu.old.eudat.logic.services.operations.DatabaseRepository; import eu.old.eudat.models.data.datasetwizard.DatasetWizardModel; import eu.old.eudat.models.data.dmp.DataManagementPlan; import eu.old.eudat.queryable.QueryableList; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.fieldset.BaseFieldSet; import gr.cite.tools.logging.LoggerService; import jakarta.persistence.EntityManager; import jakarta.xml.bind.JAXBException; import org.json.JSONObject; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.time.Instant; import java.util.*; import java.util.stream.Collectors; @Service public class DatasetMigrationService { private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class)); private final DatabaseRepository databaseRepository; private final JsonHandlingService jsonHandlingService; private final QueryFactory queryFactory; private final XmlHandlingService xmlHandlingService; private static final int PageSize = 500; private static final boolean TestMode = false; private final EntityManager entityManager; public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager) { this.databaseRepository = databaseRepository; this.jsonHandlingService = jsonHandlingService; this.queryFactory = queryFactory; this.xmlHandlingService = xmlHandlingService; this.entityManager = entityManager; } public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { DatasetDao datasetDao = databaseRepository.getDatasetDao(); long total = datasetDao.asQueryable().count(); logger.debug("Migrate Dataset Total : " + total); int page = 0; List items; do { items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); if (items != null && !items.isEmpty()) { logger.debug("Migrate Dataset " + page * PageSize + " of " + total); List dmpBlueprints = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().map(x-> x.getDmp().getProfile().getId()).distinct().toList()).collect(); Map dmpBlueprintsMap = new HashMap<>(); for (DmpBlueprintEntity dmpBlueprint : dmpBlueprints) { DefinitionEntity definitionEntity = this.xmlHandlingService.fromXml(DefinitionEntity.class, dmpBlueprint.getDefinition()); dmpBlueprintsMap.put(dmpBlueprint.getId(), definitionEntity); } List descriptionTemplates = this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collect(); Map creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId));; for (Dataset item : items) { //entityManager.detach(item); //DatasetWizardModel model = new DatasetWizardModel(); //model.fromDataModel(item); DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null); if (definition == null || definition.getSections() == null || definition.getSections().size() < item.getDmpSectionIndex()) { logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); } UUID sectionId = definition.getSections().get(item.getDmpSectionIndex()).getId(); if (sectionId == null) { logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); } List itemDescriptionTemplates = descriptionTemplates.stream().filter(x-> x.getDescriptionTemplateId().equals(item.getProfile().getId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList(); if (itemDescriptionTemplates.isEmpty()) { logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); if(descriptionTemplates.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) continue; throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); } if (itemDescriptionTemplates.size() > 1) { logger.error("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); //TODO CLEAN Duplicates dmp, DescriptionTemplate, SectionId } DescriptionEntity data = new DescriptionEntity(); data.setId(item.getId()); data.setDescription(item.getDescription()); if (item.getCreator() != null) { data.setCreatedById(item.getCreator().getId()); } else { data.setCreatedById(creatorsByDmp.getOrDefault(item.getDmp().getId(), null)); } data.setDmpId(item.getDmp().getId()); data.setLabel(item.getLabel()); data.setDmpDescriptionTemplateId(itemDescriptionTemplates.get(0).getId()); data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now()); data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now()); if (item.getFinalizedAt() != null) data.setFinalizedAt(item.getFinalizedAt().toInstant()); if (item.getStatus() == 99) { data.setIsActive(IsActive.Inactive); data.setStatus(DescriptionStatus.Canceled); } else { data.setIsActive(IsActive.Active); data.setStatus(DescriptionStatus.of(item.getStatus())); } if (item.getProperties() != null) { JSONObject jObject = new JSONObject(item.getProperties()); Map properties = jObject.toMap(); PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity(); List fieldEntities = new ArrayList<>(); for (String key: properties.keySet() ) { FieldEntity fieldEntity = new FieldEntity(); fieldEntity.setKey(key); fieldEntity.setValue(properties.get(key) != null ? properties.get(key).toString(): null); fieldEntities.add(fieldEntity); } propertyDefinitionEntity.setFields(fieldEntities); data.setProperties(this.jsonHandlingService.toJson(propertyDefinitionEntity)); } if (data.getCreatedById() == null){//TODO logger.error("Migration skipped creator not found " + item.getId()); throw new MyApplicationException("Migration skipped creator not found " + item.getId()); } this.entityManager.persist(data); this.entityManager.flush(); } page++; } } while (items != null && !items.isEmpty() && !TestMode); } }