add user contact migration

This commit is contained in:
Efstratios Giannopoulos 2023-11-24 17:18:28 +02:00
parent e698b198e7
commit c6f10b6a29
10 changed files with 165 additions and 41 deletions

View File

@ -4,7 +4,7 @@ import eu.eudat.commons.enums.DmpAccessType;
import eu.eudat.commons.enums.DmpStatus; import eu.eudat.commons.enums.DmpStatus;
import eu.eudat.commons.enums.DmpVersionStatus; import eu.eudat.commons.enums.DmpVersionStatus;
import eu.eudat.commons.enums.IsActive; import eu.eudat.commons.enums.IsActive;
import eu.eudat.data.converters.enums.DmpAccessTypeConverter; import eu.eudat.data.converters.enums.DmpAccessTypeNullableConverter;
import eu.eudat.data.converters.enums.DmpStatusConverter; import eu.eudat.data.converters.enums.DmpStatusConverter;
import eu.eudat.data.converters.enums.DmpVersionStatusConverter; import eu.eudat.data.converters.enums.DmpVersionStatusConverter;
import eu.eudat.data.converters.enums.IsActiveConverter; import eu.eudat.data.converters.enums.IsActiveConverter;
@ -41,13 +41,13 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _versionStatus = "versionStatus"; public static final String _versionStatus = "versionStatus";
@Column(name = "status", nullable = false) @Column(name = "status")
@Convert(converter = DmpStatusConverter.class) @Convert(converter = DmpStatusConverter.class)
private DmpStatus status; private DmpStatus status;
public static final String _status = "status"; public static final String _status = "status";
@Column(name = "properties") @Column(name = "properties", nullable = false)
private String properties; private String properties;
public static final String _properties = "properties"; public static final String _properties = "properties";
@ -57,7 +57,7 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _groupId = "groupId"; public static final String _groupId = "groupId";
@Column(name = "description") @Column(name = "description", nullable = false)
private String description; private String description;
public static final String _description = "description"; public static final String _description = "description";
@ -72,7 +72,7 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _updatedAt = "updatedAt"; public static final String _updatedAt = "updatedAt";
@Column(name = "is_active", nullable = false) @Column(name = "is_active")
@Convert(converter = IsActiveConverter.class) @Convert(converter = IsActiveConverter.class)
private IsActive isActive; private IsActive isActive;
@ -89,7 +89,7 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _creatorId = "creatorId"; public static final String _creatorId = "creatorId";
@Column(name = "access_type", nullable = false) @Column(name = "access_type", nullable = false)
@Convert(converter = DmpAccessTypeConverter.class) @Convert(converter = DmpAccessTypeNullableConverter.class)
private DmpAccessType accessType; private DmpAccessType accessType;
public static final String _accessType = "accessType"; public static final String _accessType = "accessType";
@ -99,12 +99,12 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _blueprintId = "blueprintId"; public static final String _blueprintId = "blueprintId";
@Column(name = "language") @Column(name = "language", nullable = false)
private String language; private String language;
public static final String _language = "language"; public static final String _language = "language";
@Column(name = "public_after") @Column(name = "public_after", nullable = false)
private Instant publicAfter; private Instant publicAfter;
public static final String _publicAfter = "publicAfter"; public static final String _publicAfter = "publicAfter";

View File

@ -1,14 +0,0 @@
package eu.eudat.data.converters.enums;
import eu.eudat.commons.enums.DmpAccessType;
import jakarta.persistence.Converter;
@Converter
public class DmpAccessTypeConverter extends DatabaseEnumConverter<DmpAccessType, Short> {
@Override
protected DmpAccessType of(Short i) {
return DmpAccessType.of(i);
}
}

View File

@ -0,0 +1,25 @@
package eu.eudat.data.converters.enums;
import eu.eudat.commons.enums.DmpAccessType;
import jakarta.persistence.Converter;
@Converter
public class DmpAccessTypeNullableConverter extends DatabaseEnumConverter<DmpAccessType, Short> {
@Override
protected DmpAccessType of(Short i) {
return DmpAccessType.of(i);
}
@Override
public Short convertToDatabaseColumn(DmpAccessType value) {
if (value == null) return null;
return value.getValue();
}
@Override
public DmpAccessType convertToEntityAttribute(Short dbData) {
return dbData == null ? null : this.of(dbData);
}
}

View File

@ -19,8 +19,8 @@ CREATE TABLE public."Dmp"
finalized_at timestamp without time zone, finalized_at timestamp without time zone,
creator uuid NOT NULL, creator uuid NOT NULL,
blueprint uuid NOT NULL, blueprint uuid NOT NULL,
language character varying(100) COLLATE pg_catalog."default" NOT NULL, language character varying(100) COLLATE pg_catalog."default",
access_type smallint NOT NULL, access_type smallint,
public_after timestamp without time zone, public_after timestamp without time zone,
version_status smallint NOT NULL DEFAULT 1, version_status smallint NOT NULL DEFAULT 1,
CONSTRAINT "Dmp_pkey" PRIMARY KEY (id), CONSTRAINT "Dmp_pkey" PRIMARY KEY (id),

View File

@ -13,15 +13,15 @@ BEGIN
END END
FROM public."UserInfo"; FROM public."UserInfo";
INSERT INTO public."UserContactInfo" ("id", "user", "created_at","ordinal", "type", "value") --INSERT INTO public."UserContactInfo" ("id", "user", "created_at","ordinal", "type", "value")
SELECT uuid_generate_v1(), "id", "created", 0, 0, "email" --SELECT uuid_generate_v1(), "id", "created", 0, 0, "email"
FROM public."UserInfo" --FROM public."UserInfo"
WHERE "email" is not null AND email != ''; --WHERE "email" is not null AND email != '';
INSERT INTO public."UserContactInfo" ("id", "user", "created_at", "ordinal", "type", "value") --INSERT INTO public."UserContactInfo" ("id", "user", "created_at", "ordinal", "type", "value")
SELECT uuid_generate_v1(), "UserId", "CreationTime", 1, 0, "Email" --SELECT uuid_generate_v1(), "UserId", "CreationTime", 1, 0, "Email"
FROM public."Credential" --FROM public."Credential"
WHERE "Email" is not null AND "Email" != ''; --WHERE "Email" is not null AND "Email" != '';
INSERT INTO public."DBVersion" VALUES ('DMPDB', '00.01.021', '2023-11-20 12:00:00.000000+02', now(), 'Sync user tables data.'); INSERT INTO public."DBVersion" VALUES ('DMPDB', '00.01.021', '2023-11-20 12:00:00.000000+02', now(), 'Sync user tables data.');

View File

@ -104,11 +104,11 @@ public class DatasetMigrationService {
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex()); throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
} }
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = descriptionTemplates.stream().filter(x-> x.getDescriptionTemplateId().equals(item.getProfile().getId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList(); List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = descriptionTemplates.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) { if (itemDescriptionTemplates.isEmpty()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
if(descriptionTemplates.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) continue; if (descriptionTemplates.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) continue;
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex()); throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
} }
@ -128,6 +128,7 @@ public class DatasetMigrationService {
data.setDmpId(item.getDmp().getId()); data.setDmpId(item.getDmp().getId());
data.setLabel(item.getLabel()); data.setLabel(item.getLabel());
data.setDmpDescriptionTemplateId(itemDescriptionTemplates.get(0).getId()); data.setDmpDescriptionTemplateId(itemDescriptionTemplates.get(0).getId());
data.setDescriptionTemplateId(item.getProfile().getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now()); data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now()); data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
if (item.getFinalizedAt() != null) if (item.getFinalizedAt() != null)
@ -157,7 +158,7 @@ public class DatasetMigrationService {
} }
if (data.getCreatedById() == null){//TODO if (data.getCreatedById() == null){
logger.error("Migration skipped creator not found " + item.getId()); logger.error("Migration skipped creator not found " + item.getId());
throw new MyApplicationException("Migration skipped creator not found " + item.getId()); throw new MyApplicationException("Migration skipped creator not found " + item.getId());
} }

View File

@ -107,7 +107,7 @@ public class DmpDatasetProfileMigrationService {
DmpDescriptionTemplateEntity data = new DmpDescriptionTemplateEntity(); DmpDescriptionTemplateEntity data = new DmpDescriptionTemplateEntity();
data.setId(UUID.randomUUID()); data.setId(UUID.randomUUID());
data.setDescriptionTemplateId(item.getDatasetprofile().getId()); data.setDescriptionTemplateGroupId(item.getDatasetprofile().getGroupId());
data.setDmpId(item.getDmp().getId()); data.setDmpId(item.getDmp().getId());
data.setCreatedAt(Instant.now()); data.setCreatedAt(Instant.now());
data.setUpdatedAt(Instant.now()); data.setUpdatedAt(Instant.now());
@ -122,7 +122,7 @@ public class DmpDatasetProfileMigrationService {
} }
} while (items != null && !items.isEmpty() && !TestMode); } while (items != null && !items.isEmpty() && !TestMode);
removeDuplicates(); //removeDuplicates();
} }
private void removeDuplicates() { private void removeDuplicates() {
@ -142,7 +142,7 @@ public class DmpDatasetProfileMigrationService {
continue; continue;
} }
if (currentEntity.getDmpId().equals(previousEntity.getDmpId()) if (currentEntity.getDmpId().equals(previousEntity.getDmpId())
&& currentEntity.getDescriptionTemplateId().equals(previousEntity.getDescriptionTemplateId()) && currentEntity.getDescriptionTemplateGroupId().equals(previousEntity.getDescriptionTemplateGroupId())
&& currentEntity.getSectionId().equals(previousEntity.getSectionId()) && currentEntity.getSectionId().equals(previousEntity.getSectionId())
) { ) {
logger.warn("Removing found duplicate DmpDescriptionTemplate entity ({}) after DmpDatasetProfiles migration", currentEntity.getId()); logger.warn("Removing found duplicate DmpDescriptionTemplate entity ({}) after DmpDatasetProfiles migration", currentEntity.getId());

View File

@ -66,10 +66,10 @@ public class DmpMigrationService {
data.setId(model.getId()); data.setId(model.getId());
data.setLabel(model.getLabel()); data.setLabel(model.getLabel());
data.setDescription(model.getDescription()); data.setDescription(model.getDescription());
if (model.getCreator() != null) { if (model.getCreator() != null && model.getCreator().getId() != null) {
data.setCreatorId(model.getCreator().getId()); data.setCreatorId(model.getCreator().getId());
} else { } else {
List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("GroupId").in(List.of(item.getGroupId()))).toList(); List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("groupId").in(List.of(item.getGroupId()))).toList();
for (DMP groupDmp: groupDmps) { for (DMP groupDmp: groupDmps) {
DataManagementPlan groupDmpModel = new DataManagementPlan(); DataManagementPlan groupDmpModel = new DataManagementPlan();
groupDmpModel.fromDataModel(groupDmp); groupDmpModel.fromDataModel(groupDmp);

View File

@ -0,0 +1,79 @@
package eu.old.eudat.migration;
import eu.eudat.commons.enums.ContactInfoType;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.UserContactInfoEntity;
import eu.old.eudat.data.dao.entities.FunderDao;
import eu.old.eudat.data.dao.entities.UserInfoDao;
import eu.old.eudat.data.entities.Credential;
import eu.old.eudat.data.entities.Funder;
import eu.old.eudat.data.entities.UserInfo;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.UUID;
@Service
public class UserContactInfoMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(UserContactInfoMigrationService.class));
private static final int PageSize = 500;
private static final boolean TestMode = false;
private final DatabaseRepository databaseRepository;
private final EntityManager entityManager;
public UserContactInfoMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) {
this.databaseRepository = databaseRepository;
this.entityManager = entityManager;
}
public void migrate(){
UserInfoDao userInfoDao = databaseRepository.getUserInfoDao();
long total = userInfoDao.asQueryable().count();
logger.debug("Migrate Contacts Total : " + total);
int page = 0;
List<UserInfo> items;
do {
items = userInfoDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Contacts " + page * PageSize + " of " + total);
for (UserInfo item : items) {
// entityManager.detach(item);
if (!item.getUserStatus().equals((short)0)) continue;
List<String> emails = new ArrayList<>();
if (item.getEmail() != null && !item.getEmail().isBlank()) emails.add(item.getEmail().toLowerCase(Locale.ROOT));
for (Credential credential: item.getCredentials()) {
if (credential.getStatus().equals(0) || credential.getEmail() == null || credential.getEmail().isBlank()) continue;
if (emails.stream().noneMatch(x-> x.equals(credential.getEmail().toLowerCase(Locale.ROOT))))
emails.add(credential.getEmail().toLowerCase(Locale.ROOT));
}
int i = 0;
for (String email: emails) {
UserContactInfoEntity data = new UserContactInfoEntity();
data.setId(UUID.randomUUID());
data.setType(ContactInfoType.Email);
data.setValue(email);
data.setUserId(item.getId());
data.setCreatedAt(item.getCreated().toInstant());
data.setOrdinal(i);
i++;
this.entityManager.persist(data);
}
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty() && !TestMode);
}
}

View File

@ -33,6 +33,7 @@ public class MigrationController {
private final DmpDatasetProfileMigrationService dmpDatasetProfileMigrationService; private final DmpDatasetProfileMigrationService dmpDatasetProfileMigrationService;
private final OrganizationMigrationService organizationMigrationService; private final OrganizationMigrationService organizationMigrationService;
private final ReferenceMigrationService referenceMigrationService; private final ReferenceMigrationService referenceMigrationService;
private final UserContactInfoMigrationService userContactInfoMigrationService;
public MigrationController( public MigrationController(
DmpMigrationService dmpMigrationService, DmpMigrationService dmpMigrationService,
@ -47,7 +48,8 @@ public class MigrationController {
ResearcherMigrationService researcherMigrationService, ResearcherMigrationService researcherMigrationService,
ServiceMigrationService serviceMigrationService, ServiceMigrationService serviceMigrationService,
OrganizationMigrationService organizationMigrationService, OrganizationMigrationService organizationMigrationService,
ReferenceMigrationService referenceMigrationService) { ReferenceMigrationService referenceMigrationService,
UserContactInfoMigrationService userContactInfoMigrationService) {
this.dmpMigrationService = dmpMigrationService; this.dmpMigrationService = dmpMigrationService;
this.datasetMigrationService = datasetMigrationService; this.datasetMigrationService = datasetMigrationService;
this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService; this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
@ -61,6 +63,37 @@ public class MigrationController {
this.serviceMigrationService = serviceMigrationService; this.serviceMigrationService = serviceMigrationService;
this.organizationMigrationService = organizationMigrationService; this.organizationMigrationService = organizationMigrationService;
this.referenceMigrationService = referenceMigrationService; this.referenceMigrationService = referenceMigrationService;
this.userContactInfoMigrationService = userContactInfoMigrationService;
}
@GetMapping("all")
@Transactional
public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.dataRepositoryMigrationService.migrate();
this.externalDatasetMigrationService.migrate();
this.funderMigrationService.migrate();
this.grantMigrationService.migrate();
this.organizationMigrationService.migrate();
this.projectMigrationService.migrate();
this.registryMigrationService.migrate();
this.researcherMigrationService.migrate();
this.serviceMigrationService.migrate();
this.dmpMigrationService.migrate();
this.dmpDatasetProfileMigrationService.migrate();
this.datasetMigrationService.migrate();
this.userContactInfoMigrationService.migrate();
return true;
}
@GetMapping("contacts")
@Transactional
public boolean migrateContacts() {
this.userContactInfoMigrationService.migrate();
return true;
} }
@GetMapping("dmps") @GetMapping("dmps")