argos/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java

package eu.old.eudat.migration;

import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.data.DescriptionTemplateEntity;
import eu.eudat.data.DmpBlueprintEntity;
import eu.eudat.data.DmpDescriptionTemplateEntity;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.model.DmpBlueprint;
import eu.eudat.query.DescriptionTemplateQuery;
import eu.eudat.query.DmpBlueprintQuery;
import eu.eudat.query.DmpDescriptionTemplateQuery;
import eu.old.eudat.data.dao.entities.DmpDatasetProfileDao;
import eu.old.eudat.data.dao.entities.ResearcherDao;
import eu.old.eudat.data.entities.DMPDatasetProfile;
import eu.old.eudat.data.entities.Researcher;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import eu.old.eudat.queryable.QueryableList;
import gr.cite.tools.data.query.Paging;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;
import eu.eudat.commons.types.dmpblueprint.DefinitionEntity;

import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.time.Instant;
import java.util.*;

@Service
public class DmpDatasetProfileMigrationService {

    private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class));
    private final DatabaseRepository databaseRepository;

    private final JsonHandlingService jsonHandlingService;
    private final QueryFactory queryFactory;
    private final XmlHandlingService xmlHandlingService;
    private static final int PageSize = 500;
    private static final boolean TestMode = false;
    private final EntityManager entityManager;

    public DmpDatasetProfileMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager) {
        this.databaseRepository = databaseRepository;
        this.jsonHandlingService = jsonHandlingService;
        this.queryFactory = queryFactory;
        this.xmlHandlingService = xmlHandlingService;
        this.entityManager = entityManager;
    }

    public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
        DmpDatasetProfileDao dmpDatasetProfileDao = databaseRepository.getDmpDatasetProfileDao();
        long total = dmpDatasetProfileDao.asQueryable().count();
        logger.debug("Migrate DmpDatasetProfile Total : " + total);
        int page = 0;

        List<DMPDatasetProfile> items;
        do {
            items = dmpDatasetProfileDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
            if (items != null && !items.isEmpty()) {
                logger.debug("Migrate DmpDatasetProfile " + page * PageSize + " of " + total);

                List<DmpBlueprintEntity> dmpBlueprints = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().map(x-> x.getDmp().getProfile().getId()).distinct().toList()).collect();
                Map<UUID, DefinitionEntity> dmpBlueprintsMap = new HashMap<>();
                for (DmpBlueprintEntity dmpBlueprint : dmpBlueprints) {
                    DefinitionEntity definitionEntity = this.xmlHandlingService.fromXml(DefinitionEntity.class, dmpBlueprint.getDefinition());
                    dmpBlueprintsMap.put(dmpBlueprint.getId(), definitionEntity);
                }
                
                for (DMPDatasetProfile item : items) {
                    entityManager.detach(item);

                    DmpDatasetProfileData profileData = jsonHandlingService.fromJson(DmpDatasetProfileData.class, item.getData());

                    if (profileData == null || profileData.dmpSectionIndex == null || profileData.dmpSectionIndex.isEmpty()){
                        logger.error("Migrate DmpDatasetProfile " + item.getId() + " failed no section info ");
                        throw  new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed no section info ");
                    }
                    DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
                    if (definition == null){
                        logger.error("Migrate DmpDatasetProfile " + item.getId() + " failed blueprint definition not found for blueprint " + item.getDmp().getProfile().getId());
                        throw  new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " failed blueprint definition not found for blueprint " + item.getDmp().getProfile().getId());
                    }

                    for (int sectionIndex: profileData.dmpSectionIndex) {
                        if (definition.getSections() == null || definition.getSections().size() < sectionIndex) {
                            logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section  " + sectionIndex);
                            throw  new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section  " + sectionIndex);
                        }
                        UUID sectionId = definition.getSections().get(sectionIndex).getId();
                        if (sectionId == null) {
                            logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section  " + sectionIndex);
                            throw  new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section  " + sectionIndex);
                        }

                        DmpDescriptionTemplateEntity data = new DmpDescriptionTemplateEntity();
                        data.setId(UUID.randomUUID());
                        data.setDescriptionTemplateGroupId(item.getDatasetprofile().getGroupId());
                        data.setDmpId(item.getDmp().getId());
                        data.setCreatedAt(Instant.now());
                        data.setUpdatedAt(Instant.now());
                        data.setSectionId(sectionId);
                        data.setIsActive(IsActive.Active);
                        this.entityManager.persist(data);
                    }
                }
                this.entityManager.flush();

                page++;
            }
        } while (items != null && !items.isEmpty() && !TestMode);

        //removeDuplicates();
    }

    private void removeDuplicates() {
        logger.debug("Checking for duplicates on DmpDescriptionTemplate table after migration");
        int page = 0;

        DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery = this.queryFactory.query(DmpDescriptionTemplateQuery.class);
        List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities;
        do {
            dmpDescriptionTemplateQuery.setPage(new Paging(page * PageSize, PageSize));
            dmpDescriptionTemplateEntities = dmpDescriptionTemplateQuery.collect();

            DmpDescriptionTemplateEntity previousEntity = null;
            for(DmpDescriptionTemplateEntity currentEntity : dmpDescriptionTemplateEntities) {
                if (previousEntity == null) {
                    previousEntity = currentEntity;
                    continue;
                }
                if (currentEntity.getDmpId().equals(previousEntity.getDmpId())
                        && currentEntity.getDescriptionTemplateGroupId().equals(previousEntity.getDescriptionTemplateGroupId())
                        && currentEntity.getSectionId().equals(previousEntity.getSectionId())
                ) {
                    logger.warn("Removing found duplicate DmpDescriptionTemplate entity ({}) after DmpDatasetProfiles migration", currentEntity.getId());
                    this.entityManager.remove(currentEntity);
                } else {
                    previousEntity = currentEntity;
                }
            }
            entityManager.flush();

            page++;
        } while (!dmpDescriptionTemplateEntities.isEmpty() && !TestMode);
    }
    
    @JsonIgnoreProperties({"validationErrorModel"})
    public static class DmpDatasetProfileData {

        private List<Integer> dmpSectionIndex;

        public List<Integer> getDmpSectionIndex() {
            return dmpSectionIndex;
        }

        public void setDmpSectionIndex(List<Integer> dmpSectionIndex) {
            this.dmpSectionIndex = dmpSectionIndex;
        }

    }

}