Merge branch 'dmp-refactoring' of https://code-repo.d4science.org/MaDgiK-CITE/argos into dmp-refactoring

dmp-backend/core/src/main/java/eu/eudat/service/storage/StorageFileCleanupProperties.java

@@ -8,14 +8,14 @@ import java.util.List;
 
 @ConfigurationProperties(prefix = "storage.task")
 public class StorageFileCleanupProperties {
-	private Boolean enable;
+	private boolean enable;
 	private int intervalSeconds;
 
-	public Boolean getEnable() {
+	public boolean getEnable() {
 		return enable;
 	}
 
-	public void setEnable(Boolean enable) {
+	public void setEnable(boolean enable) {
 		this.enable = enable;
 	}
 

dmp-db-scema/updates/00.01.026_addLanguage.sql

@@ -0,0 +1,24 @@
+DO $$DECLARE
+   this_version CONSTANT varchar := '00.01.026';
+BEGIN
+   PERFORM * FROM "DBVersion" WHERE version = this_version;
+   IF FOUND THEN RETURN; END IF;
+
+        CREATE TABLE public."Language"
+        (
+            id uuid NOT NULL,
+            code character varying(20) COLLATE pg_catalog."default" NOT NULL,
+            payload text COLLATE pg_catalog."default",
+            created_at timestamp without time zone NOT NULL,
+            updated_at timestamp without time zone NOT NULL,
+            is_active smallint NOT NULL,
+            ordinal integer NOT NULL,
+            CONSTRAINT "Language_pkey" PRIMARY KEY (id)
+        )
+        WITH (
+            OIDS = FALSE
+        );
+
+        INSERT INTO public."DBVersion" VALUES ('DMPDB', '00.01.026', '2023-11-29 12:00:00.000000+02', now(), 'Add table Language.');
+
+END$$;

dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java

@@ -94,7 +94,7 @@ public class DatasetMigrationService {
 
                     DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
 
-                    if (definition == null || definition.getSections() == null || definition.getSections().size() < item.getDmpSectionIndex()) {
+                    if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) {
                         logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section  " + item.getDmpSectionIndex());
                         throw  new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section  " + item.getDmpSectionIndex());
                     }

dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java

@@ -88,7 +88,7 @@ public class DmpDatasetProfileMigrationService {
                     }
 
                     for (int sectionIndex: profileData.dmpSectionIndex) {
-                        if (definition.getSections() == null || definition.getSections().size() < sectionIndex) {
+                        if (definition.getSections() == null || definition.getSections().size() <= sectionIndex) {
                             logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section  " + sectionIndex);
                             throw  new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section  " + sectionIndex);
                         }

dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserMigrationService.java

@@ -0,0 +1,144 @@
+package eu.old.eudat.migration;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import eu.eudat.commons.JsonHandlingService;
+import eu.eudat.commons.enums.IsActive;
+import eu.eudat.commons.enums.ReferenceFieldDataType;
+import eu.eudat.commons.enums.ReferenceSourceType;
+import eu.eudat.commons.enums.ReferenceType;
+import eu.eudat.commons.types.reference.DefinitionEntity;
+import eu.eudat.commons.types.reference.FieldEntity;
+import eu.eudat.commons.types.user.AdditionalInfoEntity;
+import eu.eudat.data.ReferenceEntity;
+import eu.eudat.data.UserEntity;
+import eu.eudat.model.Reference;
+import eu.eudat.model.User;
+import eu.eudat.query.ReferenceQuery;
+import eu.eudat.query.UserQuery;
+import eu.old.eudat.data.dao.entities.UserInfoDao;
+import eu.old.eudat.data.entities.UserInfo;
+import eu.old.eudat.elastic.entities.Organization;
+import eu.old.eudat.logic.services.operations.DatabaseRepository;
+import gr.cite.tools.data.query.Ordering;
+import gr.cite.tools.data.query.Paging;
+import gr.cite.tools.data.query.QueryFactory;
+import gr.cite.tools.logging.LoggerService;
+import jakarta.persistence.EntityManager;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Service;
+
+import java.util.*;
+import java.util.stream.Collectors;
+
+@Service
+public class UserMigrationService {
+	private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(UserMigrationService.class));
+
+	private static final int PageSize = 500;
+	private static final boolean TestMode = false;
+	private final DatabaseRepository databaseRepository;
+	private final EntityManager entityManager;
+	private final QueryFactory queryFactory;
+	private final JsonHandlingService jsonHandlingService;
+	public UserMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager, QueryFactory queryFactory, JsonHandlingService jsonHandlingService) {
+		this.databaseRepository = databaseRepository;
+		this.entityManager = entityManager;
+		this.queryFactory = queryFactory;
+		this.jsonHandlingService = jsonHandlingService;
+	}
+	
+	public void migrate() throws JsonProcessingException {
+		UserInfoDao userInfoDao = databaseRepository.getUserInfoDao();
+		long total = userInfoDao.asQueryable().count();
+		logger.debug("Migrate Contacts Total : " + total);
+		int page = 0;
+
+		List<UserEntity> items;
+		List<ReferenceEntity> organizations = this.queryFactory.query(ReferenceQuery.class).types(ReferenceType.Organizations).isActive(IsActive.Active).collect();
+		
+		do {
+			UserQuery userQuery = this.queryFactory.query(UserQuery.class);
+			userQuery.setOrder(new Ordering().addDescending(User._createdAt).addDescending(User._id));
+			userQuery.setPage(new Paging(page * PageSize, PageSize));
+			items = userQuery.collect();
+			
+			if (items != null && !items.isEmpty()) {
+				logger.debug("Migrate User " + page * PageSize + " of " + total);
+				List<UserEntity> finalItems = items;
+				Map<UUID, UserInfo> oldUsers = userInfoDao.asQueryable().where((builder, root) -> root.get("id").in(finalItems.stream().map(UserEntity::getId).toList())).toList().stream().collect(Collectors.toMap(UserInfo::getId, x-> x));
+				
+				for (UserEntity item : items) {
+					UserInfo oldUser  = oldUsers.getOrDefault(item.getId(), null);
+					String avatarUrl = "";
+					String culture = "en-001";
+					String language = "en";
+					String timezone = "en";
+					String roleOrganization = "";
+					UUID organizationId = null;
+					if (oldUser != null){
+						try {
+							avatarUrl = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("avatarUrl").asText() : "";
+						} catch (Exception ignored) {
+						}
+						try {
+							culture = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("culture").get("name").asText() : "";
+						} catch (Exception ignored) {
+						}
+						try {
+							language = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("language").get("value").asText() : "";
+						} catch (Exception ignored) {
+						}
+						try {
+							timezone = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("timezone").asText() : "";
+						} catch (Exception ignored) {
+						}
+						try {
+							roleOrganization = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("roleOrganization").asText() : "";
+						} catch (Exception ignored) {
+						}
+						try {
+							String organizationReference = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("organization").get("reference").asText() : "";
+							String[] referenceParts = organizationReference.split("::", 2);
+							if (referenceParts.length == 2){
+								organizationId = organizations.stream().filter(x-> x.getReference().equalsIgnoreCase(referenceParts[1]) && x.getSource().equalsIgnoreCase(referenceParts[0])).map(ReferenceEntity::getId).findFirst().orElse(null);
+								if (organizationId == null){
+									String organizationName = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("organization").get("name").asText() : "";
+									if (organizationName != null && !organizationName.isBlank()) {
+										ReferenceEntity data = new ReferenceEntity();
+										data.setId(UUID.randomUUID());
+										data.setLabel(organizationName);
+										data.setAbbreviation(null);
+										data.setIsActive(IsActive.Active);
+										data.setType(ReferenceType.Organizations);
+										data.setCreatedAt(item.getCreatedAt());
+										data.setUpdatedAt(item.getUpdatedAt());
+										data.setReference(referenceParts[1]);
+										data.setSource(referenceParts[0]);
+										data.setSourceType(ReferenceSourceType.External);
+										organizationId = data.getId();
+										this.entityManager.persist(data);
+									}
+								}
+							}
+						} catch (Exception ignored) {
+						}
+					}
+					
+					AdditionalInfoEntity data = new AdditionalInfoEntity();
+					data.setOrganizationId(organizationId);
+					data.setRoleOrganization(roleOrganization);
+					data.setCulture(culture);
+					data.setTimezone(timezone);
+					data.setLanguage(language);
+					data.setAvatarUrl(avatarUrl);
+					item.setAdditionalInfo(this.jsonHandlingService.toJson(data));
+					this.entityManager.persist(item);
+				}
+				this.entityManager.flush();
+				
+				page++;
+			}
+		} while (items != null && !items.isEmpty() && !TestMode);
+	}
+}

dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/datasetwizard/DatasetWizardModel.java

@@ -224,7 +224,7 @@ public class DatasetWizardModel implements DataModel<Dataset, DatasetWizardModel
         this.reference = entity.getReference();
         this.description = entity.getDescription();
         this.profile = new DatasetProfileOverviewModel();
-        this.profile = this.profile.fromDataModel(entity.getProfile());
+        this.profile = entity.getProfile() == null ? null : this.profile.fromDataModel(entity.getProfile());
         this.uri = entity.getUri();
         this.dmpSectionIndex = entity.getDmpSectionIndex();
         this.registries = entity.getRegistries() != null ? entity.getRegistries().stream().map(item -> new eu.old.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()) : new ArrayList<>();

dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/userinfo/UserListingModel.java

@@ -104,7 +104,7 @@ public class UserListingModel implements DataModel<UserInfo, UserListingModel> {
         this.created = entity.getCreated();
         this.lastloggedin = entity.getLastloggedin();
        // this.additionalinfo = entity.getAdditionalinfo();
-        this.appRoles = entity.getUserRoles().stream().map(item -> item.getRole()).collect(Collectors.toList());
+       // this.appRoles = entity.getUserRoles().stream().map(item -> item.getRole()).collect(Collectors.toList());
         if (entity.getAdditionalinfo() != null) {
             try {
                 Map<String, Object> additionalInfo = new ObjectMapper().readValue(entity.getAdditionalinfo(), HashMap.class);

dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java

@@ -35,6 +35,7 @@ public class MigrationController {
     private final ReferenceMigrationService referenceMigrationService;
     private final UserContactInfoMigrationService userContactInfoMigrationService;
     private final DmpUserMigrationService dmpUserMigrationService;
+    private final UserMigrationService userMigrationService;
     
     public MigrationController(
             DmpMigrationService dmpMigrationService,
@@ -51,7 +52,7 @@ public class MigrationController {
             OrganizationMigrationService organizationMigrationService,
             ReferenceMigrationService referenceMigrationService,
             UserContactInfoMigrationService userContactInfoMigrationService,
-            DmpUserMigrationService dmpUserMigrationService) {
+            DmpUserMigrationService dmpUserMigrationService, UserMigrationService userMigrationService) {
         this.dmpMigrationService = dmpMigrationService;
         this.datasetMigrationService = datasetMigrationService;
         this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
@@ -67,6 +68,7 @@ public class MigrationController {
         this.referenceMigrationService = referenceMigrationService;
         this.userContactInfoMigrationService = userContactInfoMigrationService;
         this.dmpUserMigrationService = dmpUserMigrationService;
+        this.userMigrationService = userMigrationService;
     }
 
     @GetMapping("all")
@@ -92,6 +94,7 @@ public class MigrationController {
         this.referenceMigrationService.migrateDmpReferences();
 
         this.userContactInfoMigrationService.migrate();
+        this.userMigrationService.migrate();
         return true;
     }
 
@@ -103,6 +106,14 @@ public class MigrationController {
        
     }
 
+    @GetMapping("users")
+    @Transactional
+    public boolean migrateUsers() throws JsonProcessingException {
+        this.userMigrationService.migrate();
+        return true;
+
+    }
+    
 
     @GetMapping("dmps")
     @Transactional