argos/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java

425 lines
26 KiB
Java

package eu.old.eudat.migration;
import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.*;
import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
import eu.eudat.commons.types.dmp.DmpContactEntity;
import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
import eu.eudat.commons.types.dmpblueprint.ReferenceTypeFieldEntity;
import eu.eudat.commons.types.dmpblueprint.SectionEntity;
import eu.eudat.commons.types.dmpreference.DmpReferenceDataEntity;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.*;
import eu.eudat.model.DmpBlueprint;
import eu.eudat.query.DmpBlueprintQuery;
import eu.old.eudat.data.dao.entities.DMPDao;
import eu.old.eudat.data.entities.*;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import eu.old.eudat.models.data.dmp.DataManagementPlan;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class DmpMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpMigrationService.class));
private final DatabaseRepository databaseRepository;
private final JsonHandlingService jsonHandlingService;
private final EntityManager entityManager;
private final ConventionService conventionService;
private final XmlHandlingService xmlHandlingService;
private final Environment environment;
private final QueryFactory queryFactory;
private static final int PageSize = 500;
private static final boolean TestMode = false;
public DmpMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, EntityManager entityManager, ConventionService conventionService, XmlHandlingService xmlHandlingService, Environment environment, QueryFactory queryFactory) {
this.databaseRepository = databaseRepository;
this.jsonHandlingService = jsonHandlingService;
this.entityManager = entityManager;
this.conventionService = conventionService;
this.xmlHandlingService = xmlHandlingService;
this.environment = environment;
this.queryFactory = queryFactory;
}
public void migrate() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
DMPDao dmpDao = databaseRepository.getDmpDao();
long total = dmpDao.asQueryable().count();
logger.debug("Migrate Dmp Total : " + total);
int page = 0;
Map<String, UUID> licenseIdByName = new HashMap<>();
List<DMP> items;
do {
items = dmpDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
List<DMP> finalItems = items;
List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("groupId").in(finalItems.stream().map(DMP::getGroupId).distinct().collect(Collectors.toList()))).toList();
Map<UUID, List<DMP>> groupDmpMap = new HashMap<>();
for (DMP dmp: groupDmps) {
if (!groupDmpMap.containsKey(dmp.getGroupId())) groupDmpMap.put(dmp.getGroupId(), new ArrayList<>());
groupDmpMap.get(dmp.getGroupId()).add(dmp);
}
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dmp " + page * PageSize + " of " + total);
List<DmpBlueprintEntity> dmpBlueprintEntities = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().filter(x-> x.getProfile() != null).map(x-> x.getProfile().getId()).distinct().collect(Collectors.toList())).collectAs(new BaseFieldSet().ensure(DmpBlueprint._definition).ensure(DmpBlueprint._id));
Map<UUID, List<ReferenceTypeFieldEntity>> referenceTypeFieldsByDmpBlueprintId = new HashMap<>();
for (DmpBlueprintEntity dmpBlueprintEntity : dmpBlueprintEntities) {
eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition = this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition());
List<ReferenceTypeFieldEntity> referenceTypeFieldEntities = definition.getSections().stream().filter(x-> x.getFields() != null).map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x)
.filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.License) || x.getReferenceTypeId().equals(ReferenceTypeIds.Project) || x.getReferenceTypeId().equals(ReferenceTypeIds.Researcher)
|| x.getReferenceTypeId().equals(ReferenceTypeIds.Grants) || x.getReferenceTypeId().equals(ReferenceTypeIds.Organizations) || x.getReferenceTypeId().equals(ReferenceTypeIds.Funder))
.collect(Collectors.toList());
referenceTypeFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , referenceTypeFieldEntities);
}
for (DMP item : items) {
//entityManager.detach(item);
DataManagementPlan model = new DataManagementPlan();
model.fromDataModel(item);
List<DMP> itemGroupDmps = groupDmpMap.get(item.getGroupId());
DMP currentVersionDmp = itemGroupDmps.stream().filter(x-> DmpStatus.Finalized.getValue().equals(x.getStatus())).max(Comparator.comparing(DMP::getVersion)).orElse(null);
DmpPropertiesEntity dmpProperties = new DmpPropertiesEntity();
dmpProperties.setDmpBlueprintValues(new ArrayList<>());
dmpProperties.setContacts(new ArrayList<>());
DmpEntity data = new DmpEntity();
data.setId(model.getId());
data.setLabel(model.getLabel());
data.setDescription(model.getDescription());
if (model.getCreator() != null && model.getCreator().getId() != null) {
data.setCreatorId(model.getCreator().getId());
} else {
for (DMP groupDmp: itemGroupDmps) {
DataManagementPlan groupDmpModel = new DataManagementPlan();
groupDmpModel.fromDataModel(groupDmp);
if (groupDmpModel.getCreator() != null) {
data.setCreatorId(groupDmpModel.getCreator().getId());
break;
}
}
}
data.setGroupId(model.getGroupId());
data.setVersion((short) model.getVersion());
if (currentVersionDmp != null && currentVersionDmp.getId().equals(item.getId())){
data.setVersionStatus(DmpVersionStatus.Current);
} else {
if (DmpStatus.Finalized.getValue().equals(item.getStatus())){
if (currentVersionDmp == null) throw new MyApplicationException("Found Finalized Dmp out of the group ");
data.setVersionStatus(DmpVersionStatus.Previous);
} else {
data.setVersionStatus(DmpVersionStatus.NotFinalized);
}
}
if (model.getCreated() != null)
data.setCreatedAt(model.getCreated().toInstant());
if (model.getModified() != null)
data.setUpdatedAt(model.getModified().toInstant());
if (model.getFinalized() != null)
data.setFinalizedAt(model.getFinalized().toInstant());
data.setBlueprintId(model.getProfile().getId());
if (model.getExtraProperties() != null) {
if (model.getExtraProperties().containsKey("language") && model.getExtraProperties().get("language") != null)
data.setLanguage((String) model.getExtraProperties().get("language"));
if (model.getExtraProperties().containsKey("visible") && model.getExtraProperties().get("visible") != null)
data.setAccessType((boolean) model.getExtraProperties().get("visible") ? DmpAccessType.Public : DmpAccessType.Restricted);
if (model.getExtraProperties().containsKey("contact") && model.getExtraProperties().get("contact") != null) {
DmpContactEntity contactEntity = new DmpContactEntity();
contactEntity.setUserId(UUID.fromString((String)model.getExtraProperties().get("contact")));
dmpProperties.getContacts().add(contactEntity);
}
}
if (model.getProperties() != null) {
model.getProperties().forEach((key,val) -> {
DmpBlueprintValueEntity valueEntity = new DmpBlueprintValueEntity();
valueEntity.setFieldId(UUID.fromString(key));
valueEntity.setValue((String) val);
dmpProperties.getDmpBlueprintValues().add(valueEntity);
});
}
if (model.getStatus() == 99) {
data.setIsActive(IsActive.Inactive);
data.setStatus(DmpStatus.Draft);
} else {
data.setIsActive(IsActive.Active);
data.setStatus(DmpStatus.of((short) model.getStatus()));
}
data.setProperties(jsonHandlingService.toJson(dmpProperties));
if (data.getAccessType() == null && item.getStatus().equals(DmpStatus.Finalized.getValue())) {
logger.warn("AccessType not found set to default for dmp " + item.getId());
data.setAccessType(item.isPublic() ? DmpAccessType.Public : DmpAccessType.Restricted);
}
if ((data.getLanguage() == null || data.getLanguage().isBlank()) && item.getStatus().equals(DmpStatus.Finalized.getValue())) {
logger.warn("Language not found set to default for dmp " + item.getId());
data.setLanguage("en");
}
if (data.getCreatorId() == null){
throw new MyApplicationException("Migration skipped creator not found " + item.getId());
}
this.entityManager.persist(data);
this.entityManager.flush();
boolean shouldCreateOwner = item.getUsers() == null || item.getUsers().stream().noneMatch(x-> x.getUser() != null && data.getCreatorId().equals(x.getUser().getId()) && DmpUserRole.Owner.getValue().equals(x.getRole().shortValue()));
if (shouldCreateOwner) {
DmpUserEntity dmpUserEntity = new DmpUserEntity();
dmpUserEntity.setId(UUID.randomUUID());
dmpUserEntity.setDmpId(data.getId());
dmpUserEntity.setUserId(data.getCreatorId());
dmpUserEntity.setRole(DmpUserRole.Owner);
dmpUserEntity.setCreatedAt(data.getCreatedAt());
dmpUserEntity.setUpdatedAt(data.getCreatedAt());
dmpUserEntity.setIsActive(IsActive.Active);
this.entityManager.persist(dmpUserEntity);
this.entityManager.flush();
}
licenseIdByName = this.migrateLicense(item, model, licenseIdByName, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateOrganizations(item, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateResearchers(item, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateProjects(item, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateGrantAndFunder(item, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
}
page++;
}
} while (items != null && !items.isEmpty() && !TestMode);
}
private void migrateOrganizations(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
Set<Organisation> organisations = item.getOrganisations();
if (organisations == null || organisations.isEmpty()) return;
if (referenceTypeFieldEntities == null) {
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Organizations)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
throw new MyApplicationException("Migration failed Organizations field not found " + item.getId());
}
for (Organisation organisation : organisations) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(item.getId());
data.setReferenceId(organisation.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
}
private void migrateProjects(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
Project project = item.getProject();
if (project == null) return;
if (referenceTypeFieldEntities == null) {
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Project)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
throw new MyApplicationException("Migration failed Project field not found " + item.getId());
}
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(item.getId());
data.setReferenceId(project.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
this.entityManager.persist(data);
}
private void migrateResearchers(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
Set<Researcher> researchers = item.getResearchers();
if (researchers == null || researchers.isEmpty()) return;
if (referenceTypeFieldEntities == null) {
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Researcher)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
throw new MyApplicationException("Migration failed Researchers field not found " + item.getId());
}
for (Researcher researcher : researchers) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(item.getId());
data.setReferenceId(researcher.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
}
private void migrateGrantAndFunder(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
Grant grant = item.getGrant();
if (grant == null) return;
if (referenceTypeFieldEntities == null) {
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Grants)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
throw new MyApplicationException("Migration failed Grant field not found " + item.getId());
}
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(item.getId());
data.setReferenceId(grant.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
if (grant.getFunder() != null){
ReferenceTypeFieldEntity funderField = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Funder)).findFirst().orElse(null);
if (funderField == null) {
throw new MyApplicationException("Migration failed Funder field not found " + item.getId());
}
DmpReferenceEntity founder = new DmpReferenceEntity();
founder.setId(UUID.randomUUID());
founder.setDmpId(item.getId());
founder.setReferenceId(grant.getFunder().getId());
founder.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
founder.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
founder.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(funderField)));
founder.setIsActive(IsActive.Active);
this.entityManager.persist(founder);
}
}
private Map<String, UUID> migrateLicense(DMP item, DataManagementPlan model, Map<String, UUID> licenseIdByName, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities) throws JsonProcessingException {
if (model.getExtraProperties() != null) {
if (model.getExtraProperties().containsKey("license") && model.getExtraProperties().get("license") != null) {
if (referenceTypeFieldEntities == null) {
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.License)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
throw new MyApplicationException("Migration failed License field not found " + item.getId());
}
Object license = model.getExtraProperties().get("license");
HashMap<String, String> licenseMap = this.jsonHandlingService.mapFromJson(this.jsonHandlingService.toJson(license));
ReferenceEntity referenceEntity = new ReferenceEntity();
String licensePid;
if (licenseMap.containsKey("pid") && licenseMap.get("pid") != null) {
licensePid = licenseMap.get("pid");
referenceEntity.setReference(licensePid);
}
else return licenseIdByName;
boolean licenseExists = licenseIdByName.containsKey(licensePid);
if (!licenseExists) {
DefinitionEntity definitionEntity = new DefinitionEntity();
List<FieldEntity> fields = new ArrayList<>();
if (licenseMap.containsKey("name")) {
referenceEntity.setLabel(licenseMap.get("name"));
}
if (licenseMap.containsKey("pid")) {
if (!this.conventionService.isNullOrEmpty(licensePid)){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("pid");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licensePid);
fields.add(fieldEntity);
}
}
if (licenseMap.containsKey("tag")) {
if (!this.conventionService.isNullOrEmpty(licenseMap.get("tag"))){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("tag");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licenseMap.get("tag"));
fields.add(fieldEntity);
}
}
if (licenseMap.containsKey("uri")) {
if (!this.conventionService.isNullOrEmpty(licenseMap.get("uri"))){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("uri");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licenseMap.get("uri"));
fields.add(fieldEntity);
}
}
if (licenseMap.containsKey("abbreviation")) {
referenceEntity.setAbbreviation(licenseMap.get("abbreviation"));
}
if (licenseMap.containsKey("created") && licenseMap.get("created") != null) {
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("created")));
} else {
referenceEntity.setCreatedAt(item.getCreated() == null ? Instant.now() : item.getCreated().toInstant());
}
if (licenseMap.containsKey("modified") && licenseMap.get("modified") != null) {
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("modified")));
} else {
referenceEntity.setUpdatedAt(item.getModified() == null ? Instant.now() : item.getModified().toInstant());
}
referenceEntity.setId(UUID.randomUUID());
referenceEntity.setSourceType(ReferenceSourceType.External);
referenceEntity.setSource(this.environment.getProperty("migration.default-license-source", "null"));
referenceEntity.setTypeId(ReferenceTypeIds.License);
referenceEntity.setIsActive(IsActive.Active);
definitionEntity.setFields(fields);
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
licenseIdByName.put(licensePid, referenceEntity.getId());
}
DmpReferenceEntity dmpReferenceEntity = new DmpReferenceEntity();
dmpReferenceEntity.setId(UUID.randomUUID());
dmpReferenceEntity.setDmpId(item.getId());
dmpReferenceEntity.setReferenceId(licenseIdByName.get(licensePid));
dmpReferenceEntity.setCreatedAt(Instant.now());
dmpReferenceEntity.setUpdatedAt(Instant.now());
dmpReferenceEntity.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
dmpReferenceEntity.setIsActive(IsActive.Active);
if (!licenseExists) entityManager.persist(referenceEntity);
entityManager.persist(dmpReferenceEntity);
}
}
return licenseIdByName;
}
private DmpReferenceDataEntity buildDmpReferenceDataEntity(ReferenceTypeFieldEntity referenceTypeFieldEntity){
DmpReferenceDataEntity data = new DmpReferenceDataEntity();
data.setBlueprintFieldId(referenceTypeFieldEntity.getId());
return data;
}
}