From 93bbde962343214800fe0a968e459e5cf613039d Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Wed, 22 Nov 2023 17:52:04 +0200 Subject: [PATCH] migration changes --- .../DataRepositoryMigrationService.java | 71 +++++++++++ .../ExternalDatasetMigrationService.java | 72 ++++++++++++ .../migration/FunderMigrationService.java | 72 ++++++++++++ .../migration/GrantMigrationService.java | 72 ++++++++++++ .../OrganizationMigrationService.java | 14 ++- .../migration/ProjectMigrationService.java | 73 ++++++++++++ .../migration/RegistryMigrationService.java | 72 ++++++++++++ .../migration/ResearcherMigrationService.java | 74 ++++++++++++ .../migration/ServiceMigrationService.java | 72 ++++++++++++ .../migration/MigrationController.java | 110 ++++++++++++++++-- .../resources/config/application.properties | 4 +- 11 files changed, 689 insertions(+), 17 deletions(-) create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java new file mode 100644 index 000000000..81159be8b --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java @@ -0,0 +1,71 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.data.ReferenceEntity; +import eu.old.eudat.data.dao.entities.DataRepositoryDao; +import eu.old.eudat.data.entities.DataRepository; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Locale; + +@Service +public class DataRepositoryMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DataRepositoryMigrationService.class)); + + private static final int PageSize = 500; + private static final String InternalReferenceSource = "dmp"; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + public DataRepositoryMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + } + + public void migrate(){ + DataRepositoryDao DataRepositoryDao = databaseRepository.getDataRepositoryDao(); + long total = DataRepositoryDao.asQueryable().count(); + logger.debug("Migrate DataRepository Total : " + total); + int page = 0; + + List items; + do { + items = DataRepositoryDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate DataRepository " + page * PageSize + " of " + total); + for (DataRepository item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null DataRepository " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + eu.old.eudat.models.data.datarepository.DataRepositoryModel model = new eu.old.eudat.models.data.datarepository.DataRepositoryModel().fromDataModel(item); + + String[] referenceParts = item.getReference().split(":", 2); + boolean isInternal = referenceParts[0].equals(InternalReferenceSource); + + ReferenceEntity data = new ReferenceEntity(); + data.setId(item.getId()); + data.setLabel(item.getLabel()); + data.setIsActive(IsActive.Active); + data.setType(ReferenceType.DataRepositories); + data.setCreatedAt(item.getCreated().toInstant()); + data.setUpdatedAt(item.getModified().toInstant()); + if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(isInternal? ReferenceSourceType.Internal : ReferenceSourceType.External); + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java new file mode 100644 index 000000000..a2b224b42 --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java @@ -0,0 +1,72 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.data.ReferenceEntity; +import eu.old.eudat.data.dao.entities.ExternalDatasetDao; +import eu.old.eudat.data.entities.DMP; +import eu.old.eudat.data.entities.ExternalDataset; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Locale; + +@Service +public class ExternalDatasetMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(ExternalDatasetMigrationService.class)); + + private static final int PageSize = 500; + private static final String InternalReferenceSource = "dmp"; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + public ExternalDatasetMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + } + + public void migrate(){ + ExternalDatasetDao ExternalDatasetDao = databaseRepository.getExternalDatasetDao(); + long total = ExternalDatasetDao.asQueryable().count(); + logger.debug("Migrate ExternalDataset Total : " + total); + int page = 0; + + List items; + do { + items = ExternalDatasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate ExternalDataset " + page * PageSize + " of " + total); + for (ExternalDataset item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null ExternalDataset " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + eu.old.eudat.models.data.externaldataset.ExternalDatasetModel model = new eu.old.eudat.models.data.externaldataset.ExternalDatasetModel().fromDataModel(item); + + String[] referenceParts = item.getReference().split(":", 2); + boolean isInternal = referenceParts[0].equals(InternalReferenceSource); + + ReferenceEntity data = new ReferenceEntity(); + data.setId(item.getId()); + data.setLabel(item.getLabel()); + data.setIsActive(IsActive.Active); + data.setType(ReferenceType.Datasets); + data.setCreatedAt(item.getCreated().toInstant()); + data.setUpdatedAt(item.getModified().toInstant()); + if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(isInternal? ReferenceSourceType.Internal : ReferenceSourceType.External); + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java new file mode 100644 index 000000000..3c03f71cb --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java @@ -0,0 +1,72 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.data.ReferenceEntity; +import eu.old.eudat.data.dao.entities.FunderDao; +import eu.old.eudat.data.entities.DMP; +import eu.old.eudat.data.entities.Funder; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Locale; + +@Service +public class FunderMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(FunderMigrationService.class)); + + private static final int PageSize = 500; + private static final String InternalReferenceSource = "dmp"; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + public FunderMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + } + + public void migrate(){ + FunderDao FunderDao = databaseRepository.getFunderDao(); + long total = FunderDao.asQueryable().count(); + logger.debug("Migrate Funder Total : " + total); + int page = 0; + + List items; + do { + items = FunderDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate Funder " + page * PageSize + " of " + total); + for (Funder item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null Funder " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + eu.old.eudat.models.data.funder.Funder model = new eu.old.eudat.models.data.funder.Funder().fromDataModel(item); + + String[] referenceParts = item.getReference().split(":", 2); + boolean isInternal = referenceParts[0].equals(InternalReferenceSource); + + ReferenceEntity data = new ReferenceEntity(); + data.setId(item.getId()); + data.setLabel(item.getLabel()); + data.setIsActive(DMP.DMPStatus.fromInteger(item.getStatus()).equals(DMP.DMPStatus.DELETED) ? IsActive.Inactive : IsActive.Active); + data.setType(ReferenceType.Funder); + data.setCreatedAt(item.getCreated().toInstant()); + data.setUpdatedAt(item.getModified().toInstant()); + if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(Funder.FunderType.INTERNAL.getValue().equals(item.getType())? ReferenceSourceType.Internal : ReferenceSourceType.External); + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java new file mode 100644 index 000000000..a91708e9a --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java @@ -0,0 +1,72 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.data.ReferenceEntity; +import eu.old.eudat.data.dao.entities.GrantDao; +import eu.old.eudat.data.entities.DMP; +import eu.old.eudat.data.entities.Grant; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Locale; + +@Service +public class GrantMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(GrantMigrationService.class)); + + private static final int PageSize = 500; + private static final String InternalReferenceSource = "dmp"; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + public GrantMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + } + + public void migrate(){ + GrantDao GrantDao = databaseRepository.getGrantDao(); + long total = GrantDao.asQueryable().count(); + logger.debug("Migrate Grant Total : " + total); + int page = 0; + + List items; + do { + items = GrantDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate Grant " + page * PageSize + " of " + total); + for (Grant item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null Grant " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + + String[] referenceParts = item.getReference().split(":", 2); + boolean isInternal = referenceParts[0].equals(InternalReferenceSource); + + ReferenceEntity data = new ReferenceEntity(); + data.setId(item.getId()); + data.setLabel(item.getLabel()); + data.setDescription(item.getDescription()); + data.setIsActive(DMP.DMPStatus.fromInteger(item.getStatus()).equals(DMP.DMPStatus.DELETED) ? IsActive.Inactive : IsActive.Active); + data.setType(ReferenceType.Grants); + data.setCreatedAt(item.getCreated().toInstant()); + data.setUpdatedAt(item.getModified().toInstant()); + if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(isInternal ? ReferenceSourceType.Internal : ReferenceSourceType.External); + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java index d4191503b..67c29b91f 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java @@ -15,12 +15,13 @@ import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.List; +import java.util.Locale; @Service public class OrganizationMigrationService { private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(OrganizationMigrationService.class)); - private static final int PageSize = 100; + private static final int PageSize = 500; private static final String InternalReferenceSource = "Internal"; private final DatabaseRepository databaseRepository; private final EntityManager entityManager; @@ -37,16 +38,17 @@ public class OrganizationMigrationService { List items; do { - items = organisationDao.asQueryable().orderBy((builder, root) -> builder.desc(root.get("created"))).skip(page * PageSize).take(PageSize).toList(); + items = organisationDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); if (items != null && !items.isEmpty()) { logger.debug("Migrate Organisation " + page * PageSize + " of " + total); for (Organisation item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null Organisation " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + eu.old.eudat.models.data.dmp.Organisation model = new eu.old.eudat.models.data.dmp.Organisation().fromDataModel(item); - if (item.getReference() == null || !item.getReference().contains(":")){ - logger.debug("Can not migrate Organisation " + item.getId()); - continue; - } String[] referenceParts = item.getReference().split(":", 2); boolean isInternal = referenceParts[0].equals(InternalReferenceSource); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java new file mode 100644 index 000000000..fd26f5564 --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java @@ -0,0 +1,73 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.data.ReferenceEntity; +import eu.old.eudat.data.dao.entities.ProjectDao; +import eu.old.eudat.data.entities.DMP; +import eu.old.eudat.data.entities.Project; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Locale; + +@Service +public class ProjectMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(ProjectMigrationService.class)); + + private static final int PageSize = 500; + private static final String InternalReferenceSource = "dmp"; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + public ProjectMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + } + + public void migrate(){ + ProjectDao ProjectDao = databaseRepository.getProjectDao(); + long total = ProjectDao.asQueryable().count(); + logger.debug("Migrate Project Total : " + total); + int page = 0; + + List items; + do { + items = ProjectDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate Project " + page * PageSize + " of " + total); + for (Project item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null Project " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + eu.old.eudat.models.data.project.Project model = new eu.old.eudat.models.data.project.Project().fromDataModel(item); + + String[] referenceParts = item.getReference().split(":", 2); + boolean isInternal = referenceParts[0].equals(InternalReferenceSource); + + ReferenceEntity data = new ReferenceEntity(); + data.setId(item.getId()); + data.setLabel(item.getLabel()); + data.setDescription(item.getDescription()); + data.setIsActive(DMP.DMPStatus.fromInteger(item.getStatus()).equals(DMP.DMPStatus.DELETED) ? IsActive.Inactive : IsActive.Active); + data.setType(ReferenceType.Project); + data.setCreatedAt(item.getCreated().toInstant()); + data.setUpdatedAt(item.getModified().toInstant()); + if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(Project.ProjectType.INTERNAL.getValue().equals(item.getType())? ReferenceSourceType.Internal : ReferenceSourceType.External); + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java new file mode 100644 index 000000000..adca99673 --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java @@ -0,0 +1,72 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.data.ReferenceEntity; +import eu.old.eudat.data.dao.entities.RegistryDao; +import eu.old.eudat.data.entities.DMP; +import eu.old.eudat.data.entities.Registry; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Locale; + +@Service +public class RegistryMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(RegistryMigrationService.class)); + + private static final int PageSize = 500; + private static final String InternalReferenceSource = "dmp"; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + public RegistryMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + } + + public void migrate(){ + RegistryDao RegistryDao = databaseRepository.getRegistryDao(); + long total = RegistryDao.asQueryable().count(); + logger.debug("Migrate Registry Total : " + total); + int page = 0; + + List items; + do { + items = RegistryDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate Registry " + page * PageSize + " of " + total); + for (Registry item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null Registry " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + eu.old.eudat.models.data.registries.RegistryModel model = new eu.old.eudat.models.data.registries.RegistryModel().fromDataModel(item); + + String[] referenceParts = item.getReference().split(":", 2); + boolean isInternal = referenceParts[0].equals(InternalReferenceSource); + + ReferenceEntity data = new ReferenceEntity(); + data.setId(item.getId()); + data.setLabel(item.getLabel()); + data.setIsActive(DMP.DMPStatus.fromInteger(item.getStatus()).equals(DMP.DMPStatus.DELETED) ? IsActive.Inactive : IsActive.Active); + data.setType(ReferenceType.Registries); + data.setCreatedAt(item.getCreated().toInstant()); + data.setUpdatedAt(item.getModified().toInstant()); + if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(isInternal? ReferenceSourceType.Internal : ReferenceSourceType.External); + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java new file mode 100644 index 000000000..bfe97470b --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java @@ -0,0 +1,74 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.data.ReferenceEntity; +import eu.old.eudat.data.dao.entities.OrganisationDao; +import eu.old.eudat.data.dao.entities.ResearcherDao; +import eu.old.eudat.data.entities.DMP; +import eu.old.eudat.data.entities.Organisation; +import eu.old.eudat.data.entities.Researcher; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Locale; + +@Service +public class ResearcherMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(ResearcherMigrationService.class)); + + private static final int PageSize = 500; + private static final String InternalReferenceSource = "dmp"; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + public ResearcherMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + } + + public void migrate(){ + ResearcherDao researcherDao = databaseRepository.getResearcherDao(); + long total = researcherDao.asQueryable().count(); + logger.debug("Migrate Researcher Total : " + total); + int page = 0; + + List items; + do { + items = researcherDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate Researcher " + page * PageSize + " of " + total); + for (Researcher item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null Researcher " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + eu.old.eudat.models.data.dmp.Researcher model = new eu.old.eudat.models.data.dmp.Researcher().fromDataModel(item); + + String[] referenceParts = item.getReference().split(":", 2); + boolean isInternal = referenceParts[0].equals(InternalReferenceSource); + + ReferenceEntity data = new ReferenceEntity(); + data.setId(item.getId()); + data.setLabel(item.getLabel()); + data.setIsActive(DMP.DMPStatus.fromInteger(item.getStatus()).equals(DMP.DMPStatus.DELETED) ? IsActive.Inactive : IsActive.Active); + data.setType(ReferenceType.Researcher); + data.setCreatedAt(item.getCreated().toInstant()); + data.setUpdatedAt(item.getModified().toInstant()); + if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(isInternal? ReferenceSourceType.Internal : ReferenceSourceType.External); + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java new file mode 100644 index 000000000..4e5169073 --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java @@ -0,0 +1,72 @@ +package eu.old.eudat.migration; + +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.data.ReferenceEntity; +import eu.old.eudat.data.dao.entities.ServiceDao; +import eu.old.eudat.data.entities.DMP; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Locale; + +@Service +public class ServiceMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(ServiceMigrationService.class)); + + private static final int PageSize = 500; + private static final String InternalReferenceSource = "dmp"; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + public ServiceMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + } + + public void migrate(){ + ServiceDao ServiceDao = databaseRepository.getServiceDao(); + long total = ServiceDao.asQueryable().count(); + logger.debug("Migrate Service Total : " + total); + int page = 0; + + List items; + do { + items = ServiceDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); + if (items != null && !items.isEmpty()) { + logger.debug("Migrate Service " + page * PageSize + " of " + total); + for (eu.old.eudat.data.entities.Service item : items) { + if (item.getReference() == null || !item.getReference().contains(":")){ + logger.warn("Reference generated because is null Service " + item.getId()); + item.setReference(InternalReferenceSource + ":" + item.getId().toString().toLowerCase(Locale.ROOT)); + } + + eu.old.eudat.models.data.services.ServiceModel model = new eu.old.eudat.models.data.services.ServiceModel().fromDataModel(item); + String[] referenceParts = item.getReference().split(":", 2); + boolean isInternal = referenceParts[0].equals(InternalReferenceSource); + + ReferenceEntity data = new ReferenceEntity(); + data.setId(item.getId()); + data.setLabel(item.getLabel()); + data.setDescription(item.getLabel()); + data.setIsActive(DMP.DMPStatus.fromInteger(item.getStatus()).equals(DMP.DMPStatus.DELETED) ? IsActive.Inactive : IsActive.Active); + data.setType(ReferenceType.Services); + data.setCreatedAt(item.getCreated().toInstant()); + data.setUpdatedAt(item.getModified().toInstant()); + if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(isInternal? ReferenceSourceType.Internal : ReferenceSourceType.External); + this.entityManager.persist(data); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty()); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java index e892b23d1..ec8b83a4d 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java @@ -1,14 +1,8 @@ package eu.old.eudat.publicapi.migration; import com.fasterxml.jackson.core.JsonProcessingException; -import eu.eudat.commons.JsonHandlingService; -import eu.eudat.data.DmpEntity; -import eu.old.eudat.logic.services.operations.DatabaseRepository; -import eu.old.eudat.migration.DmpMigrationService; -import eu.old.eudat.migration.OrganizationMigrationService; +import eu.old.eudat.migration.*; import io.swagger.annotations.Api; -import jakarta.persistence.EntityManager; -import jakarta.persistence.PersistenceContext; import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.*; @@ -19,11 +13,38 @@ import org.springframework.web.bind.annotation.*; public class MigrationController { private final DmpMigrationService dmpMigrationService; + private final DataRepositoryMigrationService dataRepositoryMigrationService; + private final ExternalDatasetMigrationService externalDatasetMigrationService; + private final FunderMigrationService funderMigrationService; + private final GrantMigrationService grantMigrationService; + private final ProjectMigrationService projectMigrationService; + private final RegistryMigrationService registryMigrationService; + private final ResearcherMigrationService researcherMigrationService; + private final ServiceMigrationService serviceMigrationService; private final OrganizationMigrationService organizationMigrationService; - public MigrationController(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, EntityManager entityManager, DmpMigrationService dmpMigrationService, OrganizationMigrationService organizationMigrationService) { + public MigrationController( + DmpMigrationService dmpMigrationService, + DataRepositoryMigrationService dataRepositoryMigrationService, + ExternalDatasetMigrationService externalDatasetMigrationService, + FunderMigrationService funderMigrationService, + GrantMigrationService grantMigrationService, + ProjectMigrationService projectMigrationService, + RegistryMigrationService registryMigrationService, + ResearcherMigrationService researcherMigrationService, + ServiceMigrationService serviceMigrationService, + OrganizationMigrationService organizationMigrationService + ) { this.dmpMigrationService = dmpMigrationService; + this.dataRepositoryMigrationService = dataRepositoryMigrationService; + this.externalDatasetMigrationService = externalDatasetMigrationService; + this.funderMigrationService = funderMigrationService; + this.grantMigrationService = grantMigrationService; + this.projectMigrationService = projectMigrationService; + this.registryMigrationService = registryMigrationService; + this.researcherMigrationService = researcherMigrationService; + this.serviceMigrationService = serviceMigrationService; this.organizationMigrationService = organizationMigrationService; } @@ -33,11 +54,82 @@ public class MigrationController { return this.dmpMigrationService.migrate(); } - @GetMapping("organizations/migrate") + @GetMapping("organizations") @Transactional public boolean migrateOrganizations() { this.organizationMigrationService.migrate(); return true; } + @GetMapping("references") + @Transactional + public boolean migrateReferences() { + this.dataRepositoryMigrationService.migrate(); + this.externalDatasetMigrationService.migrate(); + this.funderMigrationService.migrate(); + this.grantMigrationService.migrate(); + this.organizationMigrationService.migrate(); + this.projectMigrationService.migrate(); + this.registryMigrationService.migrate(); + this.researcherMigrationService.migrate(); + this.serviceMigrationService.migrate(); + return true; + } + + @GetMapping("dataRepositories") + @Transactional + public boolean migrateDataRepositories() { + this.dataRepositoryMigrationService.migrate(); + return true; + } + + @GetMapping("externalDatasets") + @Transactional + public boolean migrateExternalDatasets() { + this.externalDatasetMigrationService.migrate(); + return true; + } + + @GetMapping("funders") + @Transactional + public boolean migrateFunders() { + this.funderMigrationService.migrate(); + return true; + } + + @GetMapping("grants") + @Transactional + public boolean migrateGrants() { + this.grantMigrationService.migrate(); + return true; + } + + @GetMapping("projects") + @Transactional + public boolean migrateProjects() { + this.projectMigrationService.migrate(); + return true; + } + + @GetMapping("registries") + @Transactional + public boolean migrateRegistries() { + this.registryMigrationService.migrate(); + return true; + } + + @GetMapping("researchers") + @Transactional + public boolean migrateResearchers() { + this.researcherMigrationService.migrate(); + return true; + } + + @GetMapping("services") + @Transactional + public boolean migrateServices() { + this.serviceMigrationService.migrate(); + return true; + } + } diff --git a/dmp-migration-tool/web/src/main/resources/config/application.properties b/dmp-migration-tool/web/src/main/resources/config/application.properties index 2ceb9e5ee..2397efe84 100644 --- a/dmp-migration-tool/web/src/main/resources/config/application.properties +++ b/dmp-migration-tool/web/src/main/resources/config/application.properties @@ -30,8 +30,8 @@ admin.mail.subject=You have been invited to the Dataset Template {templateName} ####################SPRING MAIL CONFIGURATIONS################# spring.mail.default-encoding=UTF-8 -spring.mail.host=hermes.local.cite.gr -spring.mail.port=25 +spring.mail.host= +spring.mail.port= spring.mail.protocol=smtp spring.mail.test-connection=false spring.mail.properties.mail.smtp.auth=false