package eu.old.eudat.migration; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import eu.eudat.commons.JsonHandlingService; import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.enums.IsActive; import eu.eudat.data.DmpBlueprintEntity; import eu.eudat.data.DmpDescriptionTemplateEntity; import eu.eudat.query.DmpBlueprintQuery; import eu.eudat.query.DmpDescriptionTemplateQuery; import eu.old.eudat.data.dao.entities.DmpDatasetProfileDao; import eu.old.eudat.data.entities.DMPDatasetProfile; import eu.old.eudat.logic.services.operations.DatabaseRepository; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.logging.LoggerService; import jakarta.persistence.EntityManager; import jakarta.persistence.Tuple; import jakarta.persistence.criteria.CriteriaBuilder; import jakarta.persistence.criteria.CriteriaDelete; import jakarta.persistence.criteria.CriteriaQuery; import jakarta.persistence.criteria.Root; import jakarta.xml.bind.JAXBException; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import org.xml.sax.SAXException; import eu.eudat.commons.types.dmpblueprint.DefinitionEntity; import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.security.SecureRandom; import java.time.Instant; import java.util.*; @Service public class DmpDatasetProfileMigrationService { private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class)); private final DatabaseRepository databaseRepository; private final JsonHandlingService jsonHandlingService; private final QueryFactory queryFactory; private final XmlHandlingService xmlHandlingService; private static final int PageSize = 500; private static final boolean TestMode = false; private final EntityManager entityManager; public DmpDatasetProfileMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager) { this.databaseRepository = databaseRepository; this.jsonHandlingService = jsonHandlingService; this.queryFactory = queryFactory; this.xmlHandlingService = xmlHandlingService; this.entityManager = entityManager; } public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { DmpDatasetProfileDao dmpDatasetProfileDao = databaseRepository.getDmpDatasetProfileDao(); long total = dmpDatasetProfileDao.asQueryable().count(); logger.debug("Migrate DmpDatasetProfile Total : " + total); int page = 0; List items; do { items = dmpDatasetProfileDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); if (items != null && !items.isEmpty()) { logger.debug("Migrate DmpDatasetProfile " + page * PageSize + " of " + total); List dmpBlueprints = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().map(x-> x.getDmp().getProfile().getId()).distinct().toList()).collect(); Map dmpBlueprintsMap = new HashMap<>(); for (DmpBlueprintEntity dmpBlueprint : dmpBlueprints) { DefinitionEntity definitionEntity = this.xmlHandlingService.fromXml(DefinitionEntity.class, dmpBlueprint.getDefinition()); dmpBlueprintsMap.put(dmpBlueprint.getId(), definitionEntity); } for (DMPDatasetProfile item : items) { entityManager.detach(item); DmpDatasetProfileData profileData = jsonHandlingService.fromJson(DmpDatasetProfileData.class, item.getData()); if (profileData == null || profileData.dmpSectionIndex == null || profileData.dmpSectionIndex.isEmpty()){ throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed no section info "); } DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null); if (definition == null){ throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed blueprint definition not found for blueprint " + item.getDmp().getProfile().getId()); } for (int sectionIndex: profileData.dmpSectionIndex) { if (definition.getSections() == null || definition.getSections().size() <= sectionIndex) { throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex); } UUID sectionId = definition.getSections().get(sectionIndex).getId(); if (sectionId == null) { throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex); } DmpDescriptionTemplateEntity data = new DmpDescriptionTemplateEntity(); data.setId(UUID.randomUUID()); data.setDescriptionTemplateGroupId(item.getDatasetprofile().getGroupId()); data.setDmpId(item.getDmp().getId()); data.setCreatedAt(Instant.now()); data.setUpdatedAt(Instant.now()); data.setSectionId(sectionId); data.setIsActive(IsActive.Active); this.entityManager.persist(data); } } this.entityManager.flush(); page++; } } while (items != null && !items.isEmpty() && !TestMode); removeDuplicates(); } private void removeDuplicates() { logger.debug("Checking for duplicates on DmpDescriptionTemplate table after migration"); DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery = this.queryFactory.query(DmpDescriptionTemplateQuery.class); long total = dmpDescriptionTemplateQuery.count(); logger.debug("Record count to check: {}", total); CriteriaBuilder b = this.entityManager.getCriteriaBuilder(); CriteriaQuery criteria = b.createQuery(Tuple.class); Root root = criteria.from(DmpDescriptionTemplateEntity.class); criteria.groupBy(Arrays.asList(root.get(DmpDescriptionTemplateEntity._dmpId), root.get(DmpDescriptionTemplateEntity._descriptionTemplateGroupId), root.get(DmpDescriptionTemplateEntity._sectionId))); criteria.multiselect(root.get(DmpDescriptionTemplateEntity._dmpId), root.get(DmpDescriptionTemplateEntity._descriptionTemplateGroupId), root.get(DmpDescriptionTemplateEntity._sectionId), b.count(root)); List resultList = this.entityManager.createQuery(criteria).getResultList(); List duplicatesList = resultList.stream().filter(x -> (long) x.get(3) > 1).toList(); CriteriaDelete delete = b.createCriteriaDelete(DmpDescriptionTemplateEntity.class); Root root1 = delete.from(DmpDescriptionTemplateEntity.class); for (Tuple duplicate : duplicatesList) { List duplicateEntities = dmpDescriptionTemplateQuery .dmpIds((UUID) duplicate.get(0)) .descriptionTemplateGroupIds((UUID) duplicate.get(1)) .sectionIds((UUID) duplicate.get(2)) .collect(); List toDelete = new ArrayList<>(duplicateEntities.stream().map(DmpDescriptionTemplateEntity::getId).toList()); toDelete.remove(0); delete.where(root1.get(DmpDescriptionTemplateEntity._id).in(toDelete)); this.entityManager.createQuery(delete).executeUpdate(); } entityManager.flush(); } @JsonIgnoreProperties({"validationErrorModel"}) public static class DmpDatasetProfileData { private List dmpSectionIndex; public List getDmpSectionIndex() { return dmpSectionIndex; } public void setDmpSectionIndex(List dmpSectionIndex) { this.dmpSectionIndex = dmpSectionIndex; } } }