From c60e90d6380aa95b2dbf717de79453a2cae97a2d Mon Sep 17 00:00:00 2001
From: Thomas Georgios Giannos <thgiannos@cite.gr>
Date: Fri, 24 Nov 2023 12:15:57 +0200
Subject: [PATCH] Removing possible duplicate records after DmpDatasetProfile
 migration

---
 .../DmpDatasetProfileMigrationService.java    | 34 ++++++++++++++++++-
 .../migration/MigrationController.java        |  2 +-
 2 files changed, 34 insertions(+), 2 deletions(-)

diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java
index c91a3fb3e..7fe610249 100644
--- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java
+++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java
@@ -16,12 +16,14 @@ import eu.eudat.data.ReferenceEntity;
 import eu.eudat.model.DmpBlueprint;
 import eu.eudat.query.DescriptionTemplateQuery;
 import eu.eudat.query.DmpBlueprintQuery;
+import eu.eudat.query.DmpDescriptionTemplateQuery;
 import eu.old.eudat.data.dao.entities.DmpDatasetProfileDao;
 import eu.old.eudat.data.dao.entities.ResearcherDao;
 import eu.old.eudat.data.entities.DMPDatasetProfile;
 import eu.old.eudat.data.entities.Researcher;
 import eu.old.eudat.logic.services.operations.DatabaseRepository;
 import eu.old.eudat.queryable.QueryableList;
+import gr.cite.tools.data.query.Paging;
 import gr.cite.tools.data.query.QueryFactory;
 import gr.cite.tools.exception.MyApplicationException;
 import gr.cite.tools.logging.LoggerService;
@@ -120,9 +122,39 @@ public class DmpDatasetProfileMigrationService {
             }
         } while (items != null && !items.isEmpty() && !TestMode);
 
+        removeDuplicates();
+    }
 
+    private void removeDuplicates() {
+        logger.debug("Checking for duplicates on DmpDescriptionTemplate table after migration");
+        int page = 0;
 
-        //TODO CLEAN Duplicates dmp, DescriptionTemplate, SectionId
+        DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery = this.queryFactory.query(DmpDescriptionTemplateQuery.class);
+        List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities;
+        do {
+            dmpDescriptionTemplateQuery.setPage(new Paging(page * PageSize, PageSize));
+            dmpDescriptionTemplateEntities = dmpDescriptionTemplateQuery.collect();
+
+            DmpDescriptionTemplateEntity previousEntity = null;
+            for(DmpDescriptionTemplateEntity currentEntity : dmpDescriptionTemplateEntities) {
+                if (previousEntity == null) {
+                    previousEntity = currentEntity;
+                    continue;
+                }
+                if (currentEntity.getDmpId().equals(previousEntity.getDmpId())
+                        && currentEntity.getDescriptionTemplateId().equals(previousEntity.getDescriptionTemplateId())
+                        && currentEntity.getSectionId().equals(previousEntity.getSectionId())
+                ) {
+                    logger.warn("Removing found duplicate DmpDescriptionTemplate entity ({}) after DmpDatasetProfiles migration", currentEntity.getId());
+                    this.entityManager.remove(currentEntity);
+                } else {
+                    previousEntity = currentEntity;
+                }
+            }
+            entityManager.flush();
+
+            page++;
+        } while (!dmpDescriptionTemplateEntities.isEmpty() && !TestMode);
     }
     
     @JsonIgnoreProperties({"validationErrorModel"})
diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
index df7fd0718..5940961a5 100644
--- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
+++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
@@ -81,7 +81,7 @@ public class MigrationController {
     @Transactional
     public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
         this.dmpDatasetProfileMigrationService.migrate();
-        return  true;
+        return true;
     }
 
     @GetMapping("references")