From 0decfb14740bca8cc79847a939c9b6b6e6f99b9f Mon Sep 17 00:00:00 2001 From: sgiannopoulos Date: Wed, 29 Nov 2023 16:56:25 +0200 Subject: [PATCH 1/2] migration fixes --- .../storage/StorageFileCleanupProperties.java | 6 +- .../migration/DatasetMigrationService.java | 2 +- .../DmpDatasetProfileMigrationService.java | 2 +- .../eudat/migration/UserMigrationService.java | 144 ++++++++++++++++++ .../datasetwizard/DatasetWizardModel.java | 2 +- .../data/userinfo/UserListingModel.java | 2 +- .../migration/MigrationController.java | 19 ++- 7 files changed, 166 insertions(+), 11 deletions(-) create mode 100644 dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserMigrationService.java diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/storage/StorageFileCleanupProperties.java b/dmp-backend/core/src/main/java/eu/eudat/service/storage/StorageFileCleanupProperties.java index ca972bbd1..87228bad5 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/storage/StorageFileCleanupProperties.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/storage/StorageFileCleanupProperties.java @@ -8,14 +8,14 @@ import java.util.List; @ConfigurationProperties(prefix = "storage.task") public class StorageFileCleanupProperties { - private Boolean enable; + private boolean enable; private int intervalSeconds; - public Boolean getEnable() { + public boolean getEnable() { return enable; } - public void setEnable(Boolean enable) { + public void setEnable(boolean enable) { this.enable = enable; } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java index 9b187527d..0ed26e336 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java @@ -94,7 +94,7 @@ public class DatasetMigrationService { DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null); - if (definition == null || definition.getSections() == null || definition.getSections().size() < item.getDmpSectionIndex()) { + if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) { logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java index 9f304a958..c7a5885b6 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java @@ -88,7 +88,7 @@ public class DmpDatasetProfileMigrationService { } for (int sectionIndex: profileData.dmpSectionIndex) { - if (definition.getSections() == null || definition.getSections().size() < sectionIndex) { + if (definition.getSections() == null || definition.getSections().size() <= sectionIndex) { logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex); throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex); } diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserMigrationService.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserMigrationService.java new file mode 100644 index 000000000..3886e9767 --- /dev/null +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserMigrationService.java @@ -0,0 +1,144 @@ +package eu.old.eudat.migration; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import eu.eudat.commons.JsonHandlingService; +import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.ReferenceFieldDataType; +import eu.eudat.commons.enums.ReferenceSourceType; +import eu.eudat.commons.enums.ReferenceType; +import eu.eudat.commons.types.reference.DefinitionEntity; +import eu.eudat.commons.types.reference.FieldEntity; +import eu.eudat.commons.types.user.AdditionalInfoEntity; +import eu.eudat.data.ReferenceEntity; +import eu.eudat.data.UserEntity; +import eu.eudat.model.Reference; +import eu.eudat.model.User; +import eu.eudat.query.ReferenceQuery; +import eu.eudat.query.UserQuery; +import eu.old.eudat.data.dao.entities.UserInfoDao; +import eu.old.eudat.data.entities.UserInfo; +import eu.old.eudat.elastic.entities.Organization; +import eu.old.eudat.logic.services.operations.DatabaseRepository; +import gr.cite.tools.data.query.Ordering; +import gr.cite.tools.data.query.Paging; +import gr.cite.tools.data.query.QueryFactory; +import gr.cite.tools.logging.LoggerService; +import jakarta.persistence.EntityManager; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import java.util.*; +import java.util.stream.Collectors; + +@Service +public class UserMigrationService { + private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(UserMigrationService.class)); + + private static final int PageSize = 500; + private static final boolean TestMode = false; + private final DatabaseRepository databaseRepository; + private final EntityManager entityManager; + private final QueryFactory queryFactory; + private final JsonHandlingService jsonHandlingService; + public UserMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager, QueryFactory queryFactory, JsonHandlingService jsonHandlingService) { + this.databaseRepository = databaseRepository; + this.entityManager = entityManager; + this.queryFactory = queryFactory; + this.jsonHandlingService = jsonHandlingService; + } + + public void migrate() throws JsonProcessingException { + UserInfoDao userInfoDao = databaseRepository.getUserInfoDao(); + long total = userInfoDao.asQueryable().count(); + logger.debug("Migrate Contacts Total : " + total); + int page = 0; + + List items; + List organizations = this.queryFactory.query(ReferenceQuery.class).types(ReferenceType.Organizations).isActive(IsActive.Active).collect(); + + do { + UserQuery userQuery = this.queryFactory.query(UserQuery.class); + userQuery.setOrder(new Ordering().addDescending(User._createdAt).addDescending(User._id)); + userQuery.setPage(new Paging(page * PageSize, PageSize)); + items = userQuery.collect(); + + if (items != null && !items.isEmpty()) { + logger.debug("Migrate User " + page * PageSize + " of " + total); + List finalItems = items; + Map oldUsers = userInfoDao.asQueryable().where((builder, root) -> root.get("id").in(finalItems.stream().map(UserEntity::getId).toList())).toList().stream().collect(Collectors.toMap(UserInfo::getId, x-> x)); + + for (UserEntity item : items) { + UserInfo oldUser = oldUsers.getOrDefault(item.getId(), null); + String avatarUrl = ""; + String culture = "en-001"; + String language = "en"; + String timezone = "en"; + String roleOrganization = ""; + UUID organizationId = null; + if (oldUser != null){ + try { + avatarUrl = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("avatarUrl").asText() : ""; + } catch (Exception ignored) { + } + try { + culture = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("culture").get("name").asText() : ""; + } catch (Exception ignored) { + } + try { + language = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("language").get("value").asText() : ""; + } catch (Exception ignored) { + } + try { + timezone = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("timezone").asText() : ""; + } catch (Exception ignored) { + } + try { + roleOrganization = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("roleOrganization").asText() : ""; + } catch (Exception ignored) { + } + try { + String organizationReference = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("organization").get("reference").asText() : ""; + String[] referenceParts = organizationReference.split("::", 2); + if (referenceParts.length == 2){ + organizationId = organizations.stream().filter(x-> x.getReference().equalsIgnoreCase(referenceParts[1]) && x.getSource().equalsIgnoreCase(referenceParts[0])).map(ReferenceEntity::getId).findFirst().orElse(null); + if (organizationId == null){ + String organizationName = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("organization").get("name").asText() : ""; + if (organizationName != null && !organizationName.isBlank()) { + ReferenceEntity data = new ReferenceEntity(); + data.setId(UUID.randomUUID()); + data.setLabel(organizationName); + data.setAbbreviation(null); + data.setIsActive(IsActive.Active); + data.setType(ReferenceType.Organizations); + data.setCreatedAt(item.getCreatedAt()); + data.setUpdatedAt(item.getUpdatedAt()); + data.setReference(referenceParts[1]); + data.setSource(referenceParts[0]); + data.setSourceType(ReferenceSourceType.External); + organizationId = data.getId(); + this.entityManager.persist(data); + } + } + } + } catch (Exception ignored) { + } + } + + AdditionalInfoEntity data = new AdditionalInfoEntity(); + data.setOrganizationId(organizationId); + data.setRoleOrganization(roleOrganization); + data.setCulture(culture); + data.setTimezone(timezone); + data.setLanguage(language); + data.setAvatarUrl(avatarUrl); + item.setAdditionalInfo(this.jsonHandlingService.toJson(data)); + this.entityManager.persist(item); + } + this.entityManager.flush(); + + page++; + } + } while (items != null && !items.isEmpty() && !TestMode); + } +} diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/datasetwizard/DatasetWizardModel.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/datasetwizard/DatasetWizardModel.java index e4170bdfe..56494291b 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/datasetwizard/DatasetWizardModel.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/datasetwizard/DatasetWizardModel.java @@ -224,7 +224,7 @@ public class DatasetWizardModel implements DataModel new eu.old.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()) : new ArrayList<>(); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/userinfo/UserListingModel.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/userinfo/UserListingModel.java index 6e2da060c..1d4bef440 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/userinfo/UserListingModel.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/userinfo/UserListingModel.java @@ -104,7 +104,7 @@ public class UserListingModel implements DataModel { this.created = entity.getCreated(); this.lastloggedin = entity.getLastloggedin(); // this.additionalinfo = entity.getAdditionalinfo(); - this.appRoles = entity.getUserRoles().stream().map(item -> item.getRole()).collect(Collectors.toList()); + // this.appRoles = entity.getUserRoles().stream().map(item -> item.getRole()).collect(Collectors.toList()); if (entity.getAdditionalinfo() != null) { try { Map additionalInfo = new ObjectMapper().readValue(entity.getAdditionalinfo(), HashMap.class); diff --git a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java index 4cc73b4b9..a0e7e3155 100644 --- a/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java +++ b/dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java @@ -35,7 +35,8 @@ public class MigrationController { private final ReferenceMigrationService referenceMigrationService; private final UserContactInfoMigrationService userContactInfoMigrationService; private final DmpUserMigrationService dmpUserMigrationService; - + private final UserMigrationService userMigrationService; + public MigrationController( DmpMigrationService dmpMigrationService, DatasetMigrationService datasetMigrationService, @@ -49,9 +50,9 @@ public class MigrationController { ResearcherMigrationService researcherMigrationService, ServiceMigrationService serviceMigrationService, OrganizationMigrationService organizationMigrationService, - ReferenceMigrationService referenceMigrationService, + ReferenceMigrationService referenceMigrationService, UserContactInfoMigrationService userContactInfoMigrationService, - DmpUserMigrationService dmpUserMigrationService) { + DmpUserMigrationService dmpUserMigrationService, UserMigrationService userMigrationService) { this.dmpMigrationService = dmpMigrationService; this.datasetMigrationService = datasetMigrationService; this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService; @@ -67,6 +68,7 @@ public class MigrationController { this.referenceMigrationService = referenceMigrationService; this.userContactInfoMigrationService = userContactInfoMigrationService; this.dmpUserMigrationService = dmpUserMigrationService; + this.userMigrationService = userMigrationService; } @GetMapping("all") @@ -90,8 +92,9 @@ public class MigrationController { this.referenceMigrationService.migrateDatasetReferences(); this.referenceMigrationService.migrateDmpReferences(); - + this.userContactInfoMigrationService.migrate(); + this.userMigrationService.migrate(); return true; } @@ -102,6 +105,14 @@ public class MigrationController { return true; } + + @GetMapping("users") + @Transactional + public boolean migrateUsers() throws JsonProcessingException { + this.userMigrationService.migrate(); + return true; + + } @GetMapping("dmps") From 351e2cb78d704584ce6445c122c271bd9ef5f41e Mon Sep 17 00:00:00 2001 From: Thomas Georgios Giannos Date: Wed, 29 Nov 2023 17:01:03 +0200 Subject: [PATCH 2/2] Updating pom for keycloak --- dmp-backend/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dmp-backend/pom.xml b/dmp-backend/pom.xml index 9682c9055..6467e3d88 100644 --- a/dmp-backend/pom.xml +++ b/dmp-backend/pom.xml @@ -349,6 +349,12 @@ 2.1.2 + + gr.cite + keycloak-api + 1.0.0 + +