argos/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java

package eu.old.eudat.migration;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import org.opencdmp.commons.JsonHandlingService;
import org.opencdmp.commons.XmlHandlingService;
import org.opencdmp.commons.enums.IsActive;
import org.opencdmp.data.DmpBlueprintEntity;
import org.opencdmp.data.DmpDescriptionTemplateEntity;
import org.opencdmp.query.DmpBlueprintQuery;
import org.opencdmp.query.DmpDescriptionTemplateQuery;
import eu.old.eudat.data.dao.entities.DmpDatasetProfileDao;
import eu.old.eudat.data.entities.DMPDatasetProfile;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import jakarta.persistence.Tuple;
import jakarta.persistence.criteria.CriteriaBuilder;
import jakarta.persistence.criteria.CriteriaDelete;
import jakarta.persistence.criteria.CriteriaQuery;
import jakarta.persistence.criteria.Root;
import jakarta.xml.bind.JAXBException;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;
import org.opencdmp.commons.types.dmpblueprint.DefinitionEntity;

import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.security.SecureRandom;
import java.time.Instant;
import java.util.*;

@Service
public class DmpDatasetProfileMigrationService {

    private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class));
    private final DatabaseRepository databaseRepository;

    private final JsonHandlingService jsonHandlingService;
    private final QueryFactory queryFactory;
    private final XmlHandlingService xmlHandlingService;
    private static final int PageSize = 500;
    private static final boolean TestMode = false;
    private final EntityManager entityManager;

    public DmpDatasetProfileMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager) {
        this.databaseRepository = databaseRepository;
        this.jsonHandlingService = jsonHandlingService;
        this.queryFactory = queryFactory;
        this.xmlHandlingService = xmlHandlingService;
        this.entityManager = entityManager;
    }

    public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
        DmpDatasetProfileDao dmpDatasetProfileDao = databaseRepository.getDmpDatasetProfileDao();
        long total = dmpDatasetProfileDao.asQueryable().count();
        logger.debug("Migrate DmpDatasetProfile Total : " + total);
        int page = 0;

        List<DMPDatasetProfile> items;
        do {
            items = dmpDatasetProfileDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
            if (items != null && !items.isEmpty()) {
                logger.debug("Migrate DmpDatasetProfile " + page * PageSize + " of " + total);

                List<DmpBlueprintEntity> dmpBlueprints = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().map(x-> x.getDmp().getProfile().getId()).distinct().toList()).collect();
                Map<UUID, DefinitionEntity> dmpBlueprintsMap = new HashMap<>();
                for (DmpBlueprintEntity dmpBlueprint : dmpBlueprints) {
                    DefinitionEntity definitionEntity = this.xmlHandlingService.fromXml(DefinitionEntity.class, dmpBlueprint.getDefinition());
                    dmpBlueprintsMap.put(dmpBlueprint.getId(), definitionEntity);
                }
                
                for (DMPDatasetProfile item : items) {
                    entityManager.detach(item);

                    DmpDatasetProfileData profileData = jsonHandlingService.fromJson(DmpDatasetProfileData.class, item.getData());

                    if (profileData == null || profileData.dmpSectionIndex == null || profileData.dmpSectionIndex.isEmpty()){
                        if (profileData != null && item.getDmp() != null &&  item.getDmp().getProfile() != null &&  item.getDmp().getProfile().getId() != null && item.getDmp().getProfile().getId().equals(UUID.fromString("86635178-36a6-484f-9057-a934e4eeecd5"))){
                            profileData.dmpSectionIndex = new ArrayList<>();
                            profileData.dmpSectionIndex.add(3);
                            logger.warn("Migrate DmpDatasetProfile " + item.getId() + " failed no section info set to 3");
                        } else {
                            throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed no section info ");
                        }
                    }
                    DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
                    if (definition == null){
                        throw  new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed blueprint definition not found for blueprint " + item.getDmp().getProfile().getId());
                    }

                    for (int sectionIndex: profileData.dmpSectionIndex) {
                        if (definition.getSections() == null || definition.getSections().size() <= sectionIndex) {
                            throw  new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section  " + sectionIndex);
                        }
                        UUID sectionId = definition.getSections().get(sectionIndex).getId();
                        if (sectionId == null) {
                            throw  new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section  " + sectionIndex);
                        }

                        DmpDescriptionTemplateEntity data = new DmpDescriptionTemplateEntity();
                        data.setId(UUID.randomUUID());
                        data.setDescriptionTemplateGroupId(item.getDatasetprofile().getGroupId());
                        data.setDmpId(item.getDmp().getId());
                        data.setCreatedAt(Instant.now());
                        data.setUpdatedAt(Instant.now());
                        data.setSectionId(sectionId);
                        data.setIsActive(IsActive.Active);
                        this.entityManager.persist(data);
                    }
                }
                this.entityManager.flush();

                page++;
            }
        } while (items != null && !items.isEmpty() && !TestMode);

        removeDuplicates();
    }

    private void removeDuplicates() {
        logger.debug("Checking for duplicates on DmpDescriptionTemplate table after migration");

        DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery = this.queryFactory.query(DmpDescriptionTemplateQuery.class);
        long total = dmpDescriptionTemplateQuery.count();
        logger.debug("Record count to check: {}", total);

        CriteriaBuilder b = this.entityManager.getCriteriaBuilder();
        CriteriaQuery<Tuple> criteria = b.createQuery(Tuple.class);
        Root<DmpDescriptionTemplateEntity> root = criteria.from(DmpDescriptionTemplateEntity.class);
        criteria.groupBy(Arrays.asList(root.get(DmpDescriptionTemplateEntity._dmpId), root.get(DmpDescriptionTemplateEntity._descriptionTemplateGroupId), root.get(DmpDescriptionTemplateEntity._sectionId)));
        criteria.multiselect(root.get(DmpDescriptionTemplateEntity._dmpId), root.get(DmpDescriptionTemplateEntity._descriptionTemplateGroupId), root.get(DmpDescriptionTemplateEntity._sectionId), b.count(root));
        List<Tuple> resultList = this.entityManager.createQuery(criteria).getResultList();

        List<Tuple> duplicatesList = resultList.stream().filter(x -> (long) x.get(3) > 1).toList();
        CriteriaDelete<DmpDescriptionTemplateEntity> delete = b.createCriteriaDelete(DmpDescriptionTemplateEntity.class);
        Root<DmpDescriptionTemplateEntity> root1 = delete.from(DmpDescriptionTemplateEntity.class);
        for (Tuple duplicate : duplicatesList) {
            List<DmpDescriptionTemplateEntity> duplicateEntities = dmpDescriptionTemplateQuery
                    .dmpIds((UUID) duplicate.get(0))
                    .descriptionTemplateGroupIds((UUID) duplicate.get(1))
                    .sectionIds((UUID) duplicate.get(2))
                    .collect();

            List<UUID> toDelete = new ArrayList<>(duplicateEntities.stream().map(DmpDescriptionTemplateEntity::getId).toList());
            toDelete.remove(0);
            delete.where(root1.get(DmpDescriptionTemplateEntity._id).in(toDelete));
            this.entityManager.createQuery(delete).executeUpdate();
        }

        entityManager.flush();
    }
    
    @JsonIgnoreProperties({"validationErrorModel"})
    public static class DmpDatasetProfileData {

        private List<Integer> dmpSectionIndex;

        public List<Integer> getDmpSectionIndex() {
            return dmpSectionIndex;
        }

        public void setDmpSectionIndex(List<Integer> dmpSectionIndex) {
            this.dmpSectionIndex = dmpSectionIndex;
        }

    }

}