MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Merge branch 'dmp-refactoring' of https://code-repo.d4science.org/MaDgiK-CITE/argos into dmp-refactoring

This commit is contained in:
Thomas Georgios Giannos 2023-11-29 17:01:07 +02:00
parent 351e2cb78d 0decfb1474
commit e3d00fb72e
8 changed files with 190 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dmp-backend/core/src/main/java/eu/eudat/service/storage/StorageFileCleanupProperties.java
View File

@ -8,14 +8,14 @@ import java.util.List;
@ConfigurationProperties(prefix = "storage.task")
public class StorageFileCleanupProperties {
private Boolean enable;
private boolean enable;
private int intervalSeconds;
public Boolean getEnable() {
public boolean getEnable() {
return enable;
}
public void setEnable(Boolean enable) {
public void setEnable(boolean enable) {
this.enable = enable;
}

24
dmp-db-scema/updates/00.01.026_addLanguage.sql Normal file
View File

@ -0,0 +1,24 @@
DO $$DECLARE
this_version CONSTANT varchar := '00.01.026';
BEGIN
PERFORM * FROM "DBVersion" WHERE version = this_version;
IF FOUND THEN RETURN; END IF;
CREATE TABLE public."Language"
(
id uuid NOT NULL,
code character varying(20) COLLATE pg_catalog."default" NOT NULL,
payload text COLLATE pg_catalog."default",
created_at timestamp without time zone NOT NULL,
updated_at timestamp without time zone NOT NULL,
is_active smallint NOT NULL,
ordinal integer NOT NULL,
CONSTRAINT "Language_pkey" PRIMARY KEY (id)
)
WITH (
OIDS = FALSE
);
INSERT INTO public."DBVersion" VALUES ('DMPDB', '00.01.026', '2023-11-29 12:00:00.000000+02', now(), 'Add table Language.');
END$$;

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -94,7 +94,7 @@ public class DatasetMigrationService {
DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
if (definition == null || definition.getSections() == null || definition.getSections().size() < item.getDmpSectionIndex()) {
if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpDatasetProfileMigrationService.java
View File

@ -88,7 +88,7 @@ public class DmpDatasetProfileMigrationService {
}
for (int sectionIndex: profileData.dmpSectionIndex) {
if (definition.getSections() == null || definition.getSections().size() < sectionIndex) {
if (definition.getSections() == null || definition.getSections().size() <= sectionIndex) {
logger.error("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex);
throw new MyApplicationException("Migrate DmpDatasetProfile " + item.getId() + " cannot found section id for section " + sectionIndex);
}

144
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserMigrationService.java Normal file
View File

@ -0,0 +1,144 @@
package eu.old.eudat.migration;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.commons.types.user.AdditionalInfoEntity;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.UserEntity;
import eu.eudat.model.Reference;
import eu.eudat.model.User;
import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.UserQuery;
import eu.old.eudat.data.dao.entities.UserInfoDao;
import eu.old.eudat.data.entities.UserInfo;
import eu.old.eudat.elastic.entities.Organization;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.Paging;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class UserMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(UserMigrationService.class));
private static final int PageSize = 500;
private static final boolean TestMode = false;
private final DatabaseRepository databaseRepository;
private final EntityManager entityManager;
private final QueryFactory queryFactory;
private final JsonHandlingService jsonHandlingService;
public UserMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager, QueryFactory queryFactory, JsonHandlingService jsonHandlingService) {
this.databaseRepository = databaseRepository;
this.entityManager = entityManager;
this.queryFactory = queryFactory;
this.jsonHandlingService = jsonHandlingService;
}
public void migrate() throws JsonProcessingException {
UserInfoDao userInfoDao = databaseRepository.getUserInfoDao();
long total = userInfoDao.asQueryable().count();
logger.debug("Migrate Contacts Total : " + total);
int page = 0;
List<UserEntity> items;
List<ReferenceEntity> organizations = this.queryFactory.query(ReferenceQuery.class).types(ReferenceType.Organizations).isActive(IsActive.Active).collect();
do {
UserQuery userQuery = this.queryFactory.query(UserQuery.class);
userQuery.setOrder(new Ordering().addDescending(User._createdAt).addDescending(User._id));
userQuery.setPage(new Paging(page * PageSize, PageSize));
items = userQuery.collect();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate User " + page * PageSize + " of " + total);
List<UserEntity> finalItems = items;
Map<UUID, UserInfo> oldUsers = userInfoDao.asQueryable().where((builder, root) -> root.get("id").in(finalItems.stream().map(UserEntity::getId).toList())).toList().stream().collect(Collectors.toMap(UserInfo::getId, x-> x));
for (UserEntity item : items) {
UserInfo oldUser = oldUsers.getOrDefault(item.getId(), null);
String avatarUrl = "";
String culture = "en-001";
String language = "en";
String timezone = "en";
String roleOrganization = "";
UUID organizationId = null;
if (oldUser != null){
try {
avatarUrl = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("avatarUrl").asText() : "";
} catch (Exception ignored) {
}
try {
culture = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("culture").get("name").asText() : "";
} catch (Exception ignored) {
}
try {
language = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("language").get("value").asText() : "";
} catch (Exception ignored) {
}
try {
timezone = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("timezone").asText() : "";
} catch (Exception ignored) {
}
try {
roleOrganization = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("roleOrganization").asText() : "";
} catch (Exception ignored) {
}
try {
String organizationReference = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("organization").get("reference").asText() : "";
String[] referenceParts = organizationReference.split("::", 2);
if (referenceParts.length == 2){
organizationId = organizations.stream().filter(x-> x.getReference().equalsIgnoreCase(referenceParts[1]) && x.getSource().equalsIgnoreCase(referenceParts[0])).map(ReferenceEntity::getId).findFirst().orElse(null);
if (organizationId == null){
String organizationName = oldUser.getAdditionalinfo() != null ? new ObjectMapper().readTree(oldUser.getAdditionalinfo()).get("organization").get("name").asText() : "";
if (organizationName != null && !organizationName.isBlank()) {
ReferenceEntity data = new ReferenceEntity();
data.setId(UUID.randomUUID());
data.setLabel(organizationName);
data.setAbbreviation(null);
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Organizations);
data.setCreatedAt(item.getCreatedAt());
data.setUpdatedAt(item.getUpdatedAt());
data.setReference(referenceParts[1]);
data.setSource(referenceParts[0]);
data.setSourceType(ReferenceSourceType.External);
organizationId = data.getId();
this.entityManager.persist(data);
}
}
}
} catch (Exception ignored) {
}
}
AdditionalInfoEntity data = new AdditionalInfoEntity();
data.setOrganizationId(organizationId);
data.setRoleOrganization(roleOrganization);
data.setCulture(culture);
data.setTimezone(timezone);
data.setLanguage(language);
data.setAvatarUrl(avatarUrl);
item.setAdditionalInfo(this.jsonHandlingService.toJson(data));
this.entityManager.persist(item);
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty() && !TestMode);
}
}

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/datasetwizard/DatasetWizardModel.java
View File

@ -224,7 +224,7 @@ public class DatasetWizardModel implements DataModel<Dataset, DatasetWizardModel
this.reference = entity.getReference();
this.description = entity.getDescription();
this.profile = new DatasetProfileOverviewModel();
this.profile = this.profile.fromDataModel(entity.getProfile());
this.profile = entity.getProfile() == null ? null : this.profile.fromDataModel(entity.getProfile());
this.uri = entity.getUri();
this.dmpSectionIndex = entity.getDmpSectionIndex();
this.registries = entity.getRegistries() != null ? entity.getRegistries().stream().map(item -> new eu.old.eudat.models.data.dataset.Registry().fromDataModel(item)).collect(Collectors.toList()) : new ArrayList<>();

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/models/data/userinfo/UserListingModel.java
View File

@ -104,7 +104,7 @@ public class UserListingModel implements DataModel<UserInfo, UserListingModel> {
this.created = entity.getCreated();
this.lastloggedin = entity.getLastloggedin();
// this.additionalinfo = entity.getAdditionalinfo();
this.appRoles = entity.getUserRoles().stream().map(item -> item.getRole()).collect(Collectors.toList());
// this.appRoles = entity.getUserRoles().stream().map(item -> item.getRole()).collect(Collectors.toList());
if (entity.getAdditionalinfo() != null) {
try {
Map<String, Object> additionalInfo = new ObjectMapper().readValue(entity.getAdditionalinfo(), HashMap.class);

13
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -35,6 +35,7 @@ public class MigrationController {
private final ReferenceMigrationService referenceMigrationService;
private final UserContactInfoMigrationService userContactInfoMigrationService;
private final DmpUserMigrationService dmpUserMigrationService;
private final UserMigrationService userMigrationService;
public MigrationController(
DmpMigrationService dmpMigrationService,
@ -51,7 +52,7 @@ public class MigrationController {
OrganizationMigrationService organizationMigrationService,
ReferenceMigrationService referenceMigrationService,
UserContactInfoMigrationService userContactInfoMigrationService,
DmpUserMigrationService dmpUserMigrationService) {
DmpUserMigrationService dmpUserMigrationService, UserMigrationService userMigrationService) {
this.dmpMigrationService = dmpMigrationService;
this.datasetMigrationService = datasetMigrationService;
this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
@ -67,6 +68,7 @@ public class MigrationController {
this.referenceMigrationService = referenceMigrationService;
this.userContactInfoMigrationService = userContactInfoMigrationService;
this.dmpUserMigrationService = dmpUserMigrationService;
this.userMigrationService = userMigrationService;
}
@GetMapping("all")
@ -92,6 +94,7 @@ public class MigrationController {
this.referenceMigrationService.migrateDmpReferences();
this.userContactInfoMigrationService.migrate();
this.userMigrationService.migrate();
return true;
}
@ -103,6 +106,14 @@ public class MigrationController {
}
@GetMapping("users")
@Transactional
public boolean migrateUsers() throws JsonProcessingException {
this.userMigrationService.migrate();
return true;
}
@GetMapping("dmps")
@Transactional