Adding migration for reference middle tables

dmp-migration-tool/data/src/main/java/eu/old/eudat/data/dao/entities/DatasetDataRepositoryDao.java

@@ -0,0 +1,10 @@
+package eu.old.eudat.data.dao.entities;
+
+import eu.old.eudat.data.dao.DatabaseAccessLayer;
+import eu.old.eudat.data.entities.DatasetDataRepository;
+
+import java.util.UUID;
+
+public interface DatasetDataRepositoryDao extends DatabaseAccessLayer<DatasetDataRepository, UUID> {
+
+}

dmp-migration-tool/data/src/main/java/eu/old/eudat/data/dao/entities/DatasetDataRepositoryDaoImpl.java

@@ -0,0 +1,51 @@
+package eu.old.eudat.data.dao.entities;
+
+import eu.old.eudat.data.dao.DatabaseAccess;
+import eu.old.eudat.data.dao.databaselayer.service.DatabaseService;
+import eu.old.eudat.data.entities.DatasetDataRepository;
+import eu.old.eudat.queryable.QueryableList;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Component;
+
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+
+@Component
+public class DatasetDataRepositoryDaoImpl extends DatabaseAccess<DatasetDataRepository> implements DatasetDataRepositoryDao{
+
+    public DatasetDataRepositoryDaoImpl(DatabaseService<DatasetDataRepository> databaseService) {
+        super(databaseService);
+    }
+
+    @Override
+    public DatasetDataRepository createOrUpdate(DatasetDataRepository item) {
+        return this.getDatabaseService().createOrUpdate(item,DatasetDataRepository.class);
+    }
+
+    @Override
+    public CompletableFuture<DatasetDataRepository> createOrUpdateAsync(DatasetDataRepository item) {
+        return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
+    }
+
+    @Override
+    @Async
+    public DatasetDataRepository find(UUID id) {
+        return getDatabaseService().getQueryable(DatasetDataRepository.class).where((builder, root) -> builder.equal(root.get("id"), id)).getSingle();
+    }
+
+    @Override
+    public DatasetDataRepository find(UUID id, String hint) {
+        throw new UnsupportedOperationException();
+    }
+
+
+    @Override
+    public void delete(DatasetDataRepository item) {
+        this.getDatabaseService().delete(item);
+    }
+
+    @Override
+    public QueryableList<DatasetDataRepository> asQueryable() {
+        return this.getDatabaseService().getQueryable(DatasetDataRepository.class);
+    }
+}

dmp-migration-tool/data/src/main/java/eu/old/eudat/data/dao/entities/DatasetRegistryDao.java

@@ -0,0 +1,10 @@
+package eu.old.eudat.data.dao.entities;
+
+import eu.old.eudat.data.dao.DatabaseAccessLayer;
+import eu.old.eudat.data.entities.DatasetRegistry;
+
+import java.util.UUID;
+
+public interface DatasetRegistryDao extends DatabaseAccessLayer<DatasetRegistry, UUID> {
+
+}

dmp-migration-tool/data/src/main/java/eu/old/eudat/data/dao/entities/DatasetRegistryDaoImpl.java

@@ -0,0 +1,51 @@
+package eu.old.eudat.data.dao.entities;
+
+import eu.old.eudat.data.dao.DatabaseAccess;
+import eu.old.eudat.data.dao.databaselayer.service.DatabaseService;
+import eu.old.eudat.data.entities.DatasetRegistry;
+import eu.old.eudat.queryable.QueryableList;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Component;
+
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+
+@Component
+public class DatasetRegistryDaoImpl extends DatabaseAccess<DatasetRegistry> implements DatasetRegistryDao {
+
+    public DatasetRegistryDaoImpl(DatabaseService<DatasetRegistry> databaseService) {
+        super(databaseService);
+    }
+
+    @Override
+    public DatasetRegistry createOrUpdate(DatasetRegistry item) {
+        return this.getDatabaseService().createOrUpdate(item,DatasetRegistry.class);
+    }
+
+    @Override
+    public CompletableFuture<DatasetRegistry> createOrUpdateAsync(DatasetRegistry item) {
+        return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
+    }
+
+    @Override
+    @Async
+    public DatasetRegistry find(UUID id) {
+        return getDatabaseService().getQueryable(DatasetRegistry.class).where((builder, root) -> builder.equal(root.get("id"), id)).getSingle();
+    }
+
+    @Override
+    public DatasetRegistry find(UUID id, String hint) {
+        throw new UnsupportedOperationException();
+    }
+
+
+    @Override
+    public void delete(DatasetRegistry item) {
+        this.getDatabaseService().delete(item);
+    }
+
+    @Override
+    public QueryableList<DatasetRegistry> asQueryable() {
+        return this.getDatabaseService().getQueryable(DatasetRegistry.class);
+    }
+}

dmp-migration-tool/data/src/main/java/eu/old/eudat/data/entities/DatasetRegistry.java

@@ -1,17 +1,19 @@
 package eu.old.eudat.data.entities;
 
 
+import eu.old.eudat.queryable.queryableentity.DataEntity;
 import org.hibernate.annotations.GenericGenerator;
 import org.hibernate.annotations.Type;
 
 import jakarta.persistence.*;
 import java.io.Serializable;
+import java.util.List;
 import java.util.UUID;
 
 
 @Entity
 @Table(name = "\"DatasetRegistry\"")
-public class DatasetRegistry {
+public class DatasetRegistry implements DataEntity<DatasetRegistry, UUID> {
 
     @Id
     @GeneratedValue
@@ -19,13 +21,13 @@ public class DatasetRegistry {
     @Column(name = "\"ID\"", updatable = false, nullable = false, columnDefinition = "BINARY(16)")
     private UUID id;
 
-    //DEPWARN dependency to Hibernate and PostgreSQL
+    @ManyToOne
-    @Column(name = "\"Dataset\"", nullable = false)
+    @JoinColumn(name = "\"Dataset\"", nullable = false)
-    private UUID dataset;
+    private Dataset dataset;
 
-     //DEPWARN dependency to Hibernate and PostgreSQL
+    @ManyToOne
-    @Column(name = "\"Registry\"", nullable = false)
+    @JoinColumn(name = "\"Registry\"", nullable = false)
-    private UUID registry;
+    private Registry registry;
 
     @Column(name = "\"Role\"")
     private Integer role;
@@ -41,19 +43,19 @@ public class DatasetRegistry {
         this.id = id;
     }
 
-    public UUID getDataset() {
+    public Dataset getDataset() {
         return dataset;
     }
 
-    public void setDataset(UUID dataset) {
+    public void setDataset(Dataset dataset) {
         this.dataset = dataset;
     }
 
-    public UUID getRegistry() {
+    public Registry getRegistry() {
         return registry;
     }
 
-    public void setRegistry(UUID registry) {
+    public void setRegistry(Registry registry) {
         this.registry = registry;
     }
 
@@ -72,4 +74,23 @@ public class DatasetRegistry {
     public void setData(String data) {
         this.data = data;
     }
+
+    @Override
+    public void update(DatasetRegistry entity) {
+        this.dataset = entity.getDataset();
+        this.registry = entity.getRegistry();
+        this.role = entity.getRole();
+    }
+
+    @Override
+    public UUID getKeys() {
+        return this.id;
+    }
+
+    @Override
+    public DatasetRegistry buildFromTuple(List<Tuple> tuple, List<String> fields, String base) {
+        String currentBase = base.isEmpty() ? "" : base + ".";
+        if(fields.contains(currentBase + "id")) this.id = UUID.fromString((String) tuple.get(0).get(currentBase + "id"));
+        return this;
+    }
 }

dmp-migration-tool/web/src/main/java/eu/old/eudat/logic/services/operations/DatabaseRepository.java

@@ -48,8 +48,12 @@ public interface DatabaseRepository {
 
     DatasetExternalDatasetDao getDatasetExternalDatasetDao();
 
+    DatasetDataRepositoryDao getDatasetDataRepositoryDao();
+
     DatasetServiceDao getDatasetServiceDao();
 
+    DatasetRegistryDao getDatasetRegistryDao();
+
     EmailConfirmationDao getLoginConfirmationEmailDao();
 
     ProjectDao getProjectDao();

dmp-migration-tool/web/src/main/java/eu/old/eudat/logic/services/operations/DatabaseRepositoryImpl.java

@@ -33,7 +33,9 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
     private ContentDao contentDao;
     private DMPProfileDao dmpProfileDao;
     private DatasetExternalDatasetDao datasetExternalDatasetDao;
+    private DatasetDataRepositoryDao datasetDataRepositoryDao;
     private DatasetServiceDao datasetServiceDao;
+    private DatasetRegistryDao datasetRegistryDao;
     private EmailConfirmationDao loginConfirmationEmailDao;
     private ProjectDao projectDao;
     private FunderDao funderDao;
@@ -250,6 +252,16 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
         this.datasetExternalDatasetDao = datasetExternalDatasetDao;
     }
 
+    @Override
+    public DatasetDataRepositoryDao getDatasetDataRepositoryDao() {
+        return datasetDataRepositoryDao;
+    }
+
+    @Autowired
+    public void setDatasetDataRepositoryDao(DatasetDataRepositoryDao datasetDataRepositoryDao) {
+        this.datasetDataRepositoryDao = datasetDataRepositoryDao;
+    }
+
     @Override
     public DatasetServiceDao getDatasetServiceDao() {
         return datasetServiceDao;
@@ -260,6 +272,16 @@ public class DatabaseRepositoryImpl implements DatabaseRepository {
         this.datasetServiceDao = datasetServiceDao;
     }
 
+    @Override
+    public DatasetRegistryDao getDatasetRegistryDao() {
+        return datasetRegistryDao;
+    }
+
+    @Autowired
+    public void setDatasetRegistryDao(DatasetRegistryDao datasetRegistryDao) {
+        this.datasetRegistryDao = datasetRegistryDao;
+    }
+
     @Override
     public EmailConfirmationDao getLoginConfirmationEmailDao() {
         return loginConfirmationEmailDao;

dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceMigrationService.java

@@ -0,0 +1,237 @@
+package eu.old.eudat.migration;
+
+import eu.eudat.commons.enums.IsActive;
+import eu.eudat.data.DescriptionReferenceEntity;
+import eu.eudat.data.DmpReferenceEntity;
+import eu.old.eudat.data.dao.entities.*;
+import eu.old.eudat.data.entities.*;
+import eu.old.eudat.logic.services.operations.DatabaseRepository;
+import gr.cite.tools.data.query.QueryFactory;
+import gr.cite.tools.logging.LoggerService;
+import jakarta.persistence.EntityManager;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.UUID;
+
+@Service
+public class ReferenceMigrationService {
+
+    private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class));
+    private final DatabaseRepository databaseRepository;
+    private final QueryFactory queryFactory;
+    private static final int PageSize = 500;
+    private static final boolean TestMode = false;
+    private final EntityManager entityManager;
+
+    public ReferenceMigrationService(DatabaseRepository databaseRepository, QueryFactory queryFactory, EntityManager entityManager) {
+        this.databaseRepository = databaseRepository;
+        this.queryFactory = queryFactory;
+        this.entityManager = entityManager;
+    }
+
+    public void migrateDmpReferences() {
+        migrateDmpOrganizations();
+        migrateDmpResearchers();
+    }
+
+    public void migrateDatasetReferences() {
+        migrateDatasetDataRepositories();
+        migrateDatasetExternalDatasets();
+        migrateDatasetRegistries();
+        migrateDatasetServices();
+    }
+
+    public void migrateDmpOrganizations() {
+        OrganisationDao organisationDao = databaseRepository.getOrganisationDao();
+        long total = organisationDao.asQueryable().count();
+        logger.debug("Migrate Dmp Organisation (from Organization) Total : " + total);
+        int page = 0;
+
+        List<Organisation> items;
+        do {
+            items = organisationDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
+            if (items != null && !items.isEmpty()) {
+                logger.debug("Migrate Dmp Organisation " + page * PageSize + " of " + total);
+
+                for (Organisation item : items) {
+//                    entityManager.detach(item);
+
+                    for (DMP dmp : item.getDmps()) {
+                        DmpReferenceEntity data = new DmpReferenceEntity();
+                        data.setId(UUID.randomUUID());
+                        data.setDmpId(dmp.getId());
+                        data.setReferenceId(item.getId());
+                        data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
+                        data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
+                        data.setIsActive(IsActive.Active);
+                        this.entityManager.persist(data);
+                    }
+                }
+                this.entityManager.flush();
+
+                page++;
+            }
+        } while (items != null && !items.isEmpty());
+    }
+
+    public void migrateDmpResearchers() {
+        ResearcherDao researcherDao = databaseRepository.getResearcherDao();
+        long total = researcherDao.asQueryable().count();
+        logger.debug("Migrate Dmp Researcher (from Researcher) Total : " + total);
+        int page = 0;
+
+        List<Researcher> items;
+        do {
+            items = researcherDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
+            if (items != null && !items.isEmpty()) {
+                logger.debug("Migrate Dmp Researcher " + page * PageSize + " of " + total);
+
+                for (Researcher item : items) {
+//                    entityManager.detach(item);
+
+                    for (DMP dmp : item.getdMPs()) {
+                        DmpReferenceEntity data = new DmpReferenceEntity();
+                        data.setId(UUID.randomUUID());
+                        data.setDmpId(dmp.getId());
+                        data.setReferenceId(item.getId());
+                        data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
+                        data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
+                        data.setIsActive(IsActive.Active);
+                        this.entityManager.persist(data);
+                    }
+                }
+                this.entityManager.flush();
+
+                page++;
+            }
+        } while (items != null && !items.isEmpty());
+    }
+
+    public void migrateDatasetDataRepositories() {
+        DatasetDataRepositoryDao datasetDataRepositoryDao = databaseRepository.getDatasetDataRepositoryDao();
+        long total = datasetDataRepositoryDao.asQueryable().count();
+        logger.debug("Migrate Dataset DataRepository (from DatasetDataRepository) Total : " + total);
+        int page = 0;
+
+        List<DatasetDataRepository> items;
+        do {
+            items = datasetDataRepositoryDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
+            if (items != null && !items.isEmpty()) {
+                logger.debug("Migrate Dataset DataRepository " + page * PageSize + " of " + total);
+
+                for (DatasetDataRepository item : items) {
+//                    entityManager.detach(item);
+
+                    DescriptionReferenceEntity data = new DescriptionReferenceEntity();
+                    data.setId(UUID.randomUUID());
+                    data.setDescriptionId(item.getDataset().getId());
+                    data.setReferenceId(item.getDataRepository().getId());
+                    data.setCreatedAt(item.getDataset().getCreated() != null ? item.getDataset().getCreated().toInstant() : Instant.now());
+                    data.setUpdatedAt(item.getDataset().getModified() != null ? item.getDataset().getModified().toInstant() : Instant.now());
+                    data.setIsActive(IsActive.Active);
+                    this.entityManager.persist(data);
+                }
+                this.entityManager.flush();
+
+                page++;
+            }
+        } while (items != null && !items.isEmpty());
+    }
+
+    public void migrateDatasetExternalDatasets() {
+        DatasetExternalDatasetDao datasetExternalDatasetDao = databaseRepository.getDatasetExternalDatasetDao();
+        long total = datasetExternalDatasetDao.asQueryable().count();
+        logger.debug("Migrate Dataset ExternalDataset (from DatasetExternalDataset) Total : " + total);
+        int page = 0;
+
+        List<DatasetExternalDataset> items;
+        do {
+            items = datasetExternalDatasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
+            if (items != null && !items.isEmpty()) {
+                logger.debug("Migrate Dataset ExternalDataset " + page * PageSize + " of " + total);
+
+                for (DatasetExternalDataset item : items) {
+//                    entityManager.detach(item);
+
+                    DescriptionReferenceEntity data = new DescriptionReferenceEntity();
+                    data.setId(UUID.randomUUID());
+                    data.setDescriptionId(item.getDataset().getId());
+                    data.setReferenceId(item.getExternalDataset().getId());
+                    data.setCreatedAt(item.getDataset().getCreated() != null ? item.getDataset().getCreated().toInstant() : Instant.now());
+                    data.setUpdatedAt(item.getDataset().getModified() != null ? item.getDataset().getModified().toInstant() : Instant.now());
+                    data.setIsActive(IsActive.Active);
+                    this.entityManager.persist(data);
+                }
+                this.entityManager.flush();
+
+                page++;
+            }
+        } while (items != null && !items.isEmpty());
+    }
+
+    public void migrateDatasetRegistries() {
+        DatasetRegistryDao datasetRegistryDao = databaseRepository.getDatasetRegistryDao();
+        long total = datasetRegistryDao.asQueryable().count();
+        logger.debug("Migrate Dataset Registry (from DatasetRegistry) Total : " + total);
+        int page = 0;
+
+        List<DatasetRegistry> items;
+        do {
+            items = datasetRegistryDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
+            if (items != null && !items.isEmpty()) {
+                logger.debug("Migrate Dataset Registry " + page * PageSize + " of " + total);
+
+                for (DatasetRegistry item : items) {
+//                    entityManager.detach(item);
+
+                    DescriptionReferenceEntity data = new DescriptionReferenceEntity();
+                    data.setId(UUID.randomUUID());
+                    data.setDescriptionId(item.getDataset().getId());
+                    data.setReferenceId(item.getRegistry().getId());
+                    data.setCreatedAt(item.getDataset().getCreated() != null ? item.getDataset().getCreated().toInstant() : Instant.now());
+                    data.setUpdatedAt(item.getDataset().getModified() != null ? item.getDataset().getModified().toInstant() : Instant.now());
+                    data.setIsActive(IsActive.Active);
+                    this.entityManager.persist(data);
+                }
+                this.entityManager.flush();
+
+                page++;
+            }
+        } while (items != null && !items.isEmpty());
+    }
+
+    public void migrateDatasetServices() {
+        DatasetServiceDao datasetServiceDao = databaseRepository.getDatasetServiceDao();
+        long total = datasetServiceDao.asQueryable().count();
+        logger.debug("Migrate Dataset Service (from DatasetService) Total : " + total);
+        int page = 0;
+
+        List<DatasetService> items;
+        do {
+            items = datasetServiceDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
+            if (items != null && !items.isEmpty()) {
+                logger.debug("Migrate Dataset Service " + page * PageSize + " of " + total);
+
+                for (DatasetService item : items) {
+//                    entityManager.detach(item);
+
+                    DescriptionReferenceEntity data = new DescriptionReferenceEntity();
+                    data.setId(UUID.randomUUID());
+                    data.setDescriptionId(item.getDataset().getId());
+                    data.setReferenceId(item.getService().getId());
+                    data.setCreatedAt(item.getDataset().getCreated() != null ? item.getDataset().getCreated().toInstant() : Instant.now());
+                    data.setUpdatedAt(item.getDataset().getModified() != null ? item.getDataset().getModified().toInstant() : Instant.now());
+                    data.setIsActive(IsActive.Active);
+                    this.entityManager.persist(data);
+                }
+                this.entityManager.flush();
+
+                page++;
+            }
+        } while (items != null && !items.isEmpty());
+    }
+
+}

dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java

@@ -32,6 +32,7 @@ public class MigrationController {
     private final DatasetMigrationService datasetMigrationService;
     private final DmpDatasetProfileMigrationService dmpDatasetProfileMigrationService;
     private final OrganizationMigrationService organizationMigrationService;
+    private final ReferenceMigrationService referenceMigrationService;
 
     public MigrationController(
             DmpMigrationService dmpMigrationService,
@@ -45,8 +46,8 @@ public class MigrationController {
             RegistryMigrationService registryMigrationService,
             ResearcherMigrationService researcherMigrationService,
             ServiceMigrationService serviceMigrationService,
-            OrganizationMigrationService organizationMigrationService
+            OrganizationMigrationService organizationMigrationService,
-    ) {
+            ReferenceMigrationService referenceMigrationService) {
         this.dmpMigrationService = dmpMigrationService;
         this.datasetMigrationService = datasetMigrationService;
         this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
@@ -59,6 +60,7 @@ public class MigrationController {
         this.researcherMigrationService = researcherMigrationService;
         this.serviceMigrationService = serviceMigrationService;
         this.organizationMigrationService = organizationMigrationService;
+        this.referenceMigrationService = referenceMigrationService;
     }
 
     @GetMapping("dmps")
@@ -79,11 +81,9 @@ public class MigrationController {
     @Transactional
     public boolean migrateDmpDatasetProfiles() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
         this.dmpDatasetProfileMigrationService.migrate();
-        
         return  true;
     }
 
-
     @GetMapping("references")
     @Transactional
     public boolean migrateReferences() {
@@ -99,6 +99,20 @@ public class MigrationController {
         return true;
     }
 
+    @GetMapping("dmp-references")
+    @Transactional
+    public boolean migrateDmpReferences() {
+        this.referenceMigrationService.migrateDmpReferences();
+        return true;
+    }
+
+    @GetMapping("dataset-references")
+    @Transactional
+    public boolean migrateDatasetReferences() {
+        this.referenceMigrationService.migrateDatasetReferences();
+        return true;
+    }
+    
     @GetMapping("organizations")
     @Transactional
     public boolean migrateOrganizations() {