MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

add user contact migration

This commit is contained in:
Efstratios Giannopoulos 2023-11-24 17:18:28 +02:00
parent e698b198e7
commit c6f10b6a29
10 changed files with 165 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
dmp-backend/core/src/main/java/eu/eudat/data/DmpEntity.java
View File

@ -4,7 +4,7 @@ import eu.eudat.commons.enums.DmpAccessType;
import eu.eudat.commons.enums.DmpStatus;
import eu.eudat.commons.enums.DmpVersionStatus;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.data.converters.enums.DmpAccessTypeConverter;
import eu.eudat.data.converters.enums.DmpAccessTypeNullableConverter;
import eu.eudat.data.converters.enums.DmpStatusConverter;
import eu.eudat.data.converters.enums.DmpVersionStatusConverter;
import eu.eudat.data.converters.enums.IsActiveConverter;
@ -41,13 +41,13 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _versionStatus = "versionStatus";
@Column(name = "status", nullable = false)
@Column(name = "status")
@Convert(converter = DmpStatusConverter.class)
private DmpStatus status;
public static final String _status = "status";
@Column(name = "properties")
@Column(name = "properties", nullable = false)
private String properties;
public static final String _properties = "properties";
@ -57,7 +57,7 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _groupId = "groupId";
@Column(name = "description")
@Column(name = "description", nullable = false)
private String description;
public static final String _description = "description";
@ -72,7 +72,7 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _updatedAt = "updatedAt";
@Column(name = "is_active", nullable = false)
@Column(name = "is_active")
@Convert(converter = IsActiveConverter.class)
private IsActive isActive;
@ -89,7 +89,7 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _creatorId = "creatorId";
@Column(name = "access_type", nullable = false)
@Convert(converter = DmpAccessTypeConverter.class)
@Convert(converter = DmpAccessTypeNullableConverter.class)
private DmpAccessType accessType;
public static final String _accessType = "accessType";
@ -99,12 +99,12 @@ public class DmpEntity implements DataEntity<DmpEntity, UUID> {
public static final String _blueprintId = "blueprintId";
@Column(name = "language")
@Column(name = "language", nullable = false)
private String language;
public static final String _language = "language";
@Column(name = "public_after")
@Column(name = "public_after", nullable = false)
private Instant publicAfter;
public static final String _publicAfter = "publicAfter";

14
dmp-backend/core/src/main/java/eu/eudat/data/converters/enums/DmpAccessTypeConverter.java
View File

@ -1,14 +0,0 @@
package eu.eudat.data.converters.enums;
import eu.eudat.commons.enums.DmpAccessType;
import jakarta.persistence.Converter;
@Converter
public class DmpAccessTypeConverter extends DatabaseEnumConverter<DmpAccessType, Short> {
@Override
protected DmpAccessType of(Short i) {
return DmpAccessType.of(i);
}
}

25
dmp-backend/core/src/main/java/eu/eudat/data/converters/enums/DmpAccessTypeNullableConverter.java Normal file
View File

@ -0,0 +1,25 @@
package eu.eudat.data.converters.enums;
import eu.eudat.commons.enums.DmpAccessType;
import jakarta.persistence.Converter;
@Converter
public class DmpAccessTypeNullableConverter extends DatabaseEnumConverter<DmpAccessType, Short> {
@Override
protected DmpAccessType of(Short i) {
return DmpAccessType.of(i);
}
@Override
public Short convertToDatabaseColumn(DmpAccessType value) {
if (value == null) return null;
return value.getValue();
}
@Override
public DmpAccessType convertToEntityAttribute(Short dbData) {
return dbData == null ? null : this.of(dbData);
}
}

4
dmp-db-scema/updates/00.01.005_Add_Dmp_table.sql
View File

@ -19,8 +19,8 @@ CREATE TABLE public."Dmp"
finalized_at timestamp without time zone,
creator uuid NOT NULL,
blueprint uuid NOT NULL,
language character varying(100) COLLATE pg_catalog."default" NOT NULL,
access_type smallint NOT NULL,
language character varying(100) COLLATE pg_catalog."default",
access_type smallint,
public_after timestamp without time zone,
version_status smallint NOT NULL DEFAULT 1,
CONSTRAINT "Dmp_pkey" PRIMARY KEY (id),

16
dmp-db-scema/updates/00.01.021_syncUserData.sql
View File

@ -13,15 +13,15 @@ BEGIN
END
FROM public."UserInfo";
INSERT INTO public."UserContactInfo" ("id", "user", "created_at","ordinal", "type", "value")
SELECT uuid_generate_v1(), "id", "created", 0, 0, "email"
FROM public."UserInfo"
WHERE "email" is not null AND email != '';
--INSERT INTO public."UserContactInfo" ("id", "user", "created_at","ordinal", "type", "value")
--SELECT uuid_generate_v1(), "id", "created", 0, 0, "email"
--FROM public."UserInfo"
--WHERE "email" is not null AND email != '';
INSERT INTO public."UserContactInfo" ("id", "user", "created_at", "ordinal", "type", "value")
SELECT uuid_generate_v1(), "UserId", "CreationTime", 1, 0, "Email"
FROM public."Credential"
WHERE "Email" is not null AND "Email" != '';
--INSERT INTO public."UserContactInfo" ("id", "user", "created_at", "ordinal", "type", "value")
--SELECT uuid_generate_v1(), "UserId", "CreationTime", 1, 0, "Email"
--FROM public."Credential"
--WHERE "Email" is not null AND "Email" != '';
INSERT INTO public."DBVersion" VALUES ('DMPDB', '00.01.021', '2023-11-20 12:00:00.000000+02', now(), 'Sync user tables data.');

7
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -104,11 +104,11 @@ public class DatasetMigrationService {
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = descriptionTemplates.stream().filter(x-> x.getDescriptionTemplateId().equals(item.getProfile().getId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = descriptionTemplates.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
if(descriptionTemplates.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) continue;
if (descriptionTemplates.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) continue;
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
}
@ -128,6 +128,7 @@ public class DatasetMigrationService {
data.setDmpId(item.getDmp().getId());
data.setLabel(item.getLabel());
data.setDmpDescriptionTemplateId(itemDescriptionTemplates.get(0).getId());
data.setDescriptionTemplateId(item.getProfile().getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
if (item.getFinalizedAt() != null)
@ -157,7 +158,7 @@ public class DatasetMigrationService {
}
if (data.getCreatedById() == null){//TODO
if (data.getCreatedById() == null){
logger.error("Migration skipped creator not found " + item.getId());
throw new MyApplicationException("Migration skipped creator not found " + item.getId());
}

6
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java
View File

@ -107,7 +107,7 @@ public class DmpDatasetProfileMigrationService {
DmpDescriptionTemplateEntity data = new DmpDescriptionTemplateEntity();
data.setId(UUID.randomUUID());
data.setDescriptionTemplateId(item.getDatasetprofile().getId());
data.setDescriptionTemplateGroupId(item.getDatasetprofile().getGroupId());
data.setDmpId(item.getDmp().getId());
data.setCreatedAt(Instant.now());
data.setUpdatedAt(Instant.now());
@ -122,7 +122,7 @@ public class DmpDatasetProfileMigrationService {
}
} while (items != null && !items.isEmpty() && !TestMode);
removeDuplicates();
//removeDuplicates();
}
private void removeDuplicates() {
@ -142,7 +142,7 @@ public class DmpDatasetProfileMigrationService {
continue;
}
if (currentEntity.getDmpId().equals(previousEntity.getDmpId())
&& currentEntity.getDescriptionTemplateId().equals(previousEntity.getDescriptionTemplateId())
&& currentEntity.getDescriptionTemplateGroupId().equals(previousEntity.getDescriptionTemplateGroupId())
&& currentEntity.getSectionId().equals(previousEntity.getSectionId())
) {
logger.warn("Removing found duplicate DmpDescriptionTemplate entity ({}) after DmpDatasetProfiles migration", currentEntity.getId());

4
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java
View File

@ -66,10 +66,10 @@ public class DmpMigrationService {
data.setId(model.getId());
data.setLabel(model.getLabel());
data.setDescription(model.getDescription());
if (model.getCreator() != null) {
if (model.getCreator() != null && model.getCreator().getId() != null) {
data.setCreatorId(model.getCreator().getId());
} else {
List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("GroupId").in(List.of(item.getGroupId()))).toList();
List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("groupId").in(List.of(item.getGroupId()))).toList();
for (DMP groupDmp: groupDmps) {
DataManagementPlan groupDmpModel = new DataManagementPlan();
groupDmpModel.fromDataModel(groupDmp);

79
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserContactInfoMigrationService.java Normal file
View File

@ -0,0 +1,79 @@
package eu.old.eudat.migration;
import eu.eudat.commons.enums.ContactInfoType;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.UserContactInfoEntity;
import eu.old.eudat.data.dao.entities.FunderDao;
import eu.old.eudat.data.dao.entities.UserInfoDao;
import eu.old.eudat.data.entities.Credential;
import eu.old.eudat.data.entities.Funder;
import eu.old.eudat.data.entities.UserInfo;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.UUID;
@Service
public class UserContactInfoMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(UserContactInfoMigrationService.class));
private static final int PageSize = 500;
private static final boolean TestMode = false;
private final DatabaseRepository databaseRepository;
private final EntityManager entityManager;
public UserContactInfoMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) {
this.databaseRepository = databaseRepository;
this.entityManager = entityManager;
}
public void migrate(){
UserInfoDao userInfoDao = databaseRepository.getUserInfoDao();
long total = userInfoDao.asQueryable().count();
logger.debug("Migrate Contacts Total : " + total);
int page = 0;
List<UserInfo> items;
do {
items = userInfoDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("id"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Contacts " + page * PageSize + " of " + total);
for (UserInfo item : items) {
// entityManager.detach(item);
if (!item.getUserStatus().equals((short)0)) continue;
List<String> emails = new ArrayList<>();
if (item.getEmail() != null && !item.getEmail().isBlank()) emails.add(item.getEmail().toLowerCase(Locale.ROOT));
for (Credential credential: item.getCredentials()) {
if (credential.getStatus().equals(0) || credential.getEmail() == null || credential.getEmail().isBlank()) continue;
if (emails.stream().noneMatch(x-> x.equals(credential.getEmail().toLowerCase(Locale.ROOT))))
emails.add(credential.getEmail().toLowerCase(Locale.ROOT));
}
int i = 0;
for (String email: emails) {
UserContactInfoEntity data = new UserContactInfoEntity();
data.setId(UUID.randomUUID());
data.setType(ContactInfoType.Email);
data.setValue(email);
data.setUserId(item.getId());
data.setCreatedAt(item.getCreated().toInstant());
data.setOrdinal(i);
i++;
this.entityManager.persist(data);
}
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty() && !TestMode);
}
}

35
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -33,6 +33,7 @@ public class MigrationController {
private final DmpDatasetProfileMigrationService dmpDatasetProfileMigrationService;
private final OrganizationMigrationService organizationMigrationService;
private final ReferenceMigrationService referenceMigrationService;
private final UserContactInfoMigrationService userContactInfoMigrationService;
public MigrationController(
DmpMigrationService dmpMigrationService,
@ -47,7 +48,8 @@ public class MigrationController {
ResearcherMigrationService researcherMigrationService,
ServiceMigrationService serviceMigrationService,
OrganizationMigrationService organizationMigrationService,
ReferenceMigrationService referenceMigrationService) {
ReferenceMigrationService referenceMigrationService,
UserContactInfoMigrationService userContactInfoMigrationService) {
this.dmpMigrationService = dmpMigrationService;
this.datasetMigrationService = datasetMigrationService;
this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
@ -61,6 +63,37 @@ public class MigrationController {
this.serviceMigrationService = serviceMigrationService;
this.organizationMigrationService = organizationMigrationService;
this.referenceMigrationService = referenceMigrationService;
this.userContactInfoMigrationService = userContactInfoMigrationService;
}
@GetMapping("all")
@Transactional
public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
this.dataRepositoryMigrationService.migrate();
this.externalDatasetMigrationService.migrate();
this.funderMigrationService.migrate();
this.grantMigrationService.migrate();
this.organizationMigrationService.migrate();
this.projectMigrationService.migrate();
this.registryMigrationService.migrate();
this.researcherMigrationService.migrate();
this.serviceMigrationService.migrate();
this.dmpMigrationService.migrate();
this.dmpDatasetProfileMigrationService.migrate();
this.datasetMigrationService.migrate();
this.userContactInfoMigrationService.migrate();
return true;
}
@GetMapping("contacts")
@Transactional
public boolean migrateContacts() {
this.userContactInfoMigrationService.migrate();
return true;
}
@GetMapping("dmps")