MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

reference migration changes

This commit is contained in:
Efstratios Giannopoulos 2024-01-23 17:20:46 +02:00
parent f49f6d1c04
commit d5292f5cec
7 changed files with 301 additions and 292 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dmp-backend/web/src/main/resources/config/deposit.yml
View File

@ -2,7 +2,7 @@ deposit:
sources: sources:
- url: http://localhost:8082 - url: http://localhost:8082
repositoryId: zenodo repositoryId: zenodo
issuer-url: ${ZENODO_ISSUER_URI:IDP_APIKEY_ISSUER_URI} issuer-url: ${ZENODO_ISSUER_URI:IDP_APIKEY_ISSUER_URI}/protocol/openid-connect/token
client-id: ${ZENODO_DEPOSIT_CLIENT_ID:} client-id: ${ZENODO_DEPOSIT_CLIENT_ID:}
client-secret: ${ZENODO_DEPOSIT_CLIENT_SECRET:} client-secret: ${ZENODO_DEPOSIT_CLIENT_SECRET:}
scope: ${ZENODO_DEPOSIT_SCOPE:} scope: ${ZENODO_DEPOSIT_SCOPE:}

38
dmp-migration-tool/data/src/main/java/eu/old/eudat/data/entities/EntityDoi.java
View File

@ -1,29 +1,53 @@
package eu.old.eudat.data.entities; package eu.old.eudat.data.entities;
import com.fasterxml.jackson.annotation.JsonValue;
import eu.eudat.commons.enums.EnumUtils;
import eu.eudat.data.converters.enums.DatabaseEnum;
import eu.eudat.data.converters.enums.DatabaseEnumConverter;
import eu.old.eudat.data.converters.DateToUTCConverter; import eu.old.eudat.data.converters.DateToUTCConverter;
import eu.old.eudat.data.entities.helpers.EntityBinder; import eu.old.eudat.data.entities.helpers.EntityBinder;
import eu.old.eudat.queryable.queryableentity.DataEntity; import eu.old.eudat.queryable.queryableentity.DataEntity;
import jakarta.persistence.*; import jakarta.persistence.*;
import java.util.Arrays;
import java.util.Date; import java.util.*;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@Entity @Entity
@Table(name = "\"EntityDoi\"") @Table(name = "\"EntityDoi\"")
public class EntityDoi implements DataEntity<EntityDoi, UUID> { public class EntityDoi implements DataEntity<EntityDoi, UUID> {
public enum EntityType { public enum EntityType implements DatabaseEnum<Short> {
DMP
DMP((short) 0);
private final Short value;
EntityType(Short value) {
this.value = value;
}
@Override
@JsonValue
public Short getValue() {
return value;
}
private static final Map<Short, EntityType> map = EnumUtils.getEnumValueMap(EntityType.class);
public static EntityType of(Short i) {
return map.get(i);
}
} }
@Id @Id
@Column(name = "id", updatable = false, nullable = false, columnDefinition = "BINARY(16)") @Column(name = "id", updatable = false, nullable = false, columnDefinition = "BINARY(16)")
private UUID id; private UUID id;
@Enumerated(EnumType.STRING)
@Column(name = "entity_type", nullable = false) @Column(name = "entity_type", nullable = false)
@Convert(converter = EntityTypeConverter.class)
private EntityType entityType; private EntityType entityType;
@Column(name = "repository_id", nullable = false) @Column(name = "repository_id", nullable = false)

14
dmp-migration-tool/data/src/main/java/eu/old/eudat/data/entities/EntityTypeConverter.java Normal file
View File

@ -0,0 +1,14 @@
package eu.old.eudat.data.entities;
import eu.eudat.data.converters.enums.DatabaseEnumConverter;
import jakarta.persistence.Converter;
@Converter
public class EntityTypeConverter extends DatabaseEnumConverter<EntityDoi.EntityType, Short> {
@Override
protected EntityDoi.EntityType of(Short i) {
return EntityDoi.EntityType.of(i);
}
}

2
dmp-migration-tool/elastic/src/main/java/eu/old/eudat/elastic/repository/DatasetRepository.java
View File

@ -84,7 +84,7 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().should(QueryBuilders.termsQuery("datasets.id.keyword", ids)); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().should(QueryBuilders.termsQuery("datasets.id.keyword", ids));
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery( "datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder()); NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery( "datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
searchSourceBuilder.query(nestedQueryBuilder); searchSourceBuilder.query(nestedQueryBuilder);
searchSourceBuilder.size(100000); searchSourceBuilder.size(10000);
searchRequest.source(searchSourceBuilder); searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT); SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return Arrays.stream(response.getHits().getHits()) return Arrays.stream(response.getHits().getHits())

365
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java
View File

@ -1,38 +1,44 @@
package eu.old.eudat.migration; package eu.old.eudat.migration;
import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.type.MapType;
import com.fasterxml.jackson.databind.type.TypeFactory;
import eu.eudat.commons.JsonHandlingService; import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService; import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.*; import eu.eudat.commons.enums.*;
import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity; import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
import eu.eudat.commons.types.dmp.DmpContactEntity; import eu.eudat.commons.types.dmp.DmpContactEntity;
import eu.eudat.commons.types.dmp.DmpPropertiesEntity; import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
import eu.eudat.commons.types.dmpblueprint.SectionEntity;
import eu.eudat.commons.types.dmpblueprint.SystemFieldEntity;
import eu.eudat.commons.types.dmpreference.DmpReferenceDataEntity;
import eu.eudat.commons.types.reference.DefinitionEntity; import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity; import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService; import eu.eudat.convention.ConventionService;
import eu.eudat.data.DmpBlueprintEntity;
import eu.eudat.data.DmpEntity; import eu.eudat.data.DmpEntity;
import eu.eudat.data.DmpReferenceEntity; import eu.eudat.data.DmpReferenceEntity;
import eu.eudat.data.ReferenceEntity; import eu.eudat.data.ReferenceEntity;
import eu.eudat.query.ReferenceQuery; import eu.eudat.model.DmpBlueprint;
import eu.eudat.query.DmpBlueprintQuery;
import eu.old.eudat.data.dao.entities.DMPDao; import eu.old.eudat.data.dao.entities.DMPDao;
import eu.old.eudat.data.entities.DMP; import eu.old.eudat.data.entities.*;
import eu.old.eudat.logic.services.operations.DatabaseRepository; import eu.old.eudat.logic.services.operations.DatabaseRepository;
import eu.old.eudat.models.data.dmp.DataManagementPlan; import eu.old.eudat.models.data.dmp.DataManagementPlan;
import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException; import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.logging.LoggerService; import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager; import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
import org.springframework.core.env.Environment; import org.springframework.core.env.Environment;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;
import java.lang.reflect.Field; import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.time.Instant; import java.time.Instant;
import java.util.*; import java.util.*;
import java.util.stream.Collectors;
@Service @Service
public class DmpMigrationService { public class DmpMigrationService {
@ -44,30 +50,33 @@ public class DmpMigrationService {
private final ConventionService conventionService; private final ConventionService conventionService;
private final XmlHandlingService xmlHandlingService; private final XmlHandlingService xmlHandlingService;
private final Environment environment; private final Environment environment;
private final QueryFactory queryFactory;
private static final int PageSize = 500; private static final int PageSize = 500;
private static final boolean TestMode = false; private static final boolean TestMode = false;
public DmpMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, EntityManager entityManager, ConventionService conventionService, XmlHandlingService xmlHandlingService, Environment environment) { public DmpMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, EntityManager entityManager, ConventionService conventionService, XmlHandlingService xmlHandlingService, Environment environment, QueryFactory queryFactory) {
this.databaseRepository = databaseRepository; this.databaseRepository = databaseRepository;
this.jsonHandlingService = jsonHandlingService; this.jsonHandlingService = jsonHandlingService;
this.entityManager = entityManager; this.entityManager = entityManager;
this.conventionService = conventionService; this.conventionService = conventionService;
this.xmlHandlingService = xmlHandlingService; this.xmlHandlingService = xmlHandlingService;
this.environment = environment; this.environment = environment;
this.queryFactory = queryFactory;
} }
public void migrate() throws JsonProcessingException, NoSuchFieldException, IllegalAccessException { public void migrate() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
DMPDao dmpDao = databaseRepository.getDmpDao(); DMPDao dmpDao = databaseRepository.getDmpDao();
long total = dmpDao.asQueryable().count(); long total = dmpDao.asQueryable().count();
logger.debug("Migrate Dmp Total : " + total); logger.debug("Migrate Dmp Total : " + total);
int page = 0; int page = 0;
Map<String, UUID> licenseIdByName = new HashMap<>();
List<DMP> items; List<DMP> items;
do { do {
items = dmpDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); items = dmpDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
List<DMP> finalItems = items; List<DMP> finalItems = items;
List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("groupId").in(finalItems.stream().map(DMP::getGroupId).distinct())).toList(); List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("groupId").in(finalItems.stream().map(DMP::getGroupId).distinct().collect(Collectors.toList()))).toList();
Map<UUID, List<DMP>> groupDmpMap = new HashMap<>(); Map<UUID, List<DMP>> groupDmpMap = new HashMap<>();
for (DMP dmp: groupDmps) { for (DMP dmp: groupDmps) {
if (!groupDmpMap.containsKey(dmp.getGroupId())) groupDmpMap.put(dmp.getGroupId(), new ArrayList<>()); if (!groupDmpMap.containsKey(dmp.getGroupId())) groupDmpMap.put(dmp.getGroupId(), new ArrayList<>());
@ -75,8 +84,15 @@ public class DmpMigrationService {
} }
if (items != null && !items.isEmpty()) { if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dmp " + page * PageSize + " of " + total); logger.debug("Migrate Dmp " + page * PageSize + " of " + total);
List<DmpBlueprintEntity> dmpBlueprintEntities = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().filter(x-> x.getProfile() != null).map(x-> x.getProfile().getId()).distinct().collect(Collectors.toList())).collectAs(new BaseFieldSet().ensure(DmpBlueprint._definition).ensure(DmpBlueprint._id));
Map<UUID, List<SystemFieldEntity>> systemFieldsByDmpBlueprintId = new HashMap<>();
for (DmpBlueprintEntity dmpBlueprintEntity : dmpBlueprintEntities) {
eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition =this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition());
List<SystemFieldEntity> systemFieldEntities = definition.getSections().stream().map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.System)).map(x-> (SystemFieldEntity)x).collect(Collectors.toList());
systemFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , systemFieldEntities);
}
for (DMP item : items) { for (DMP item : items) {
//entityManager.detach(item); //entityManager.detach(item);
@ -122,14 +138,14 @@ public class DmpMigrationService {
data.setAccessType((boolean) model.getExtraProperties().get("visible") ? DmpAccessType.Public : DmpAccessType.Restricted); data.setAccessType((boolean) model.getExtraProperties().get("visible") ? DmpAccessType.Public : DmpAccessType.Restricted);
if (model.getExtraProperties().containsKey("contact") && model.getExtraProperties().get("contact") != null) { if (model.getExtraProperties().containsKey("contact") && model.getExtraProperties().get("contact") != null) {
DmpContactEntity contactEntity = new DmpContactEntity(); DmpContactEntity contactEntity = new DmpContactEntity();
contactEntity.setUserId((String) model.getExtraProperties().get("contact")); contactEntity.setUserId(UUID.fromString((String)model.getExtraProperties().get("contact")));
dmpProperties.getContacts().add(contactEntity); dmpProperties.getContacts().add(contactEntity);
} }
} }
if (model.getProperties() != null) { if (model.getProperties() != null) {
model.getProperties().forEach((key,val) -> { model.getProperties().forEach((key,val) -> {
DmpBlueprintValueEntity valueEntity = new DmpBlueprintValueEntity(); DmpBlueprintValueEntity valueEntity = new DmpBlueprintValueEntity();
valueEntity.setFieldId(key); valueEntity.setFieldId(UUID.fromString(key));
valueEntity.setValue((String) val); valueEntity.setValue((String) val);
dmpProperties.getDmpBlueprintValues().add(valueEntity); dmpProperties.getDmpBlueprintValues().add(valueEntity);
}); });
@ -158,124 +174,247 @@ public class DmpMigrationService {
this.entityManager.persist(data); this.entityManager.persist(data);
this.entityManager.flush(); this.entityManager.flush();
licenseIdByName = this.migrateLicense(item, model, licenseIdByName, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateOrganizations(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateResearchers(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateProjects(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateGrantAndFunder(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
} }
page++; page++;
} }
} while (items != null && !items.isEmpty() && !TestMode); } while (items != null && !items.isEmpty() && !TestMode);
} }
private void migrateOrganizations(DMP item, List<SystemFieldEntity> systemFieldEntities){
Set<Organisation> organisations = item.getOrganisations();
if (organisations == null || organisations.isEmpty()) return;
public void migrateDmpLicenses() throws JsonProcessingException { if (systemFieldEntities == null) {
DMPDao dmpDao = databaseRepository.getDmpDao(); logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
long total = dmpDao.asQueryable().count(); throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
logger.debug("Migrate Licenses for Dmp Total : " + total); }
int page = 0; SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Organizations)).findFirst().orElse(null);
if (systemField == null) {
logger.warn("Migration failed Organizations field not found " + item.getId());
throw new MyApplicationException("Migration failed Organizations field not found " + item.getId());
}
for (Organisation organisation : organisations) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(item.getId());
data.setReferenceId(organisation.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
this.entityManager.flush();
}
Set<String> collectedLicenses = new HashSet<>(); private void migrateProjects(DMP item, List<SystemFieldEntity> systemFieldEntities){
Map<String, UUID> licenseIdByName = new HashMap<>(); Project project = item.getProject();
List<DMP> items; if (project == null) return;
do { if (systemFieldEntities == null) {
items = dmpDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList(); logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Project)).findFirst().orElse(null);
if (systemField == null) {
logger.warn("Migration failed Project field not found " + item.getId());
throw new MyApplicationException("Migration failed Project field not found " + item.getId());
}
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(item.getId());
data.setReferenceId(project.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
this.entityManager.persist(data);
this.entityManager.flush();
}
if (items != null && !items.isEmpty()) { private void migrateResearchers(DMP item, List<SystemFieldEntity> systemFieldEntities){
logger.debug("Migrate Licenses for Dmp " + page * PageSize + " of " + total); Set<Researcher> researchers = item.getResearchers();
if (researchers == null || researchers.isEmpty()) return;
if (systemFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Researchers)).findFirst().orElse(null);
if (systemField == null) {
logger.warn("Migration failed Researchers field not found " + item.getId());
throw new MyApplicationException("Migration failed Researchers field not found " + item.getId());
}
for (Researcher researcher : researchers) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(item.getId());
data.setReferenceId(researcher.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
this.entityManager.flush();
}
for (DMP item : items) { private void migrateGrantAndFunder(DMP item, List<SystemFieldEntity> systemFieldEntities){
DataManagementPlan model = new DataManagementPlan(); Grant grant = item.getGrant();
model.fromDataModel(item); if (grant == null) return;
if (model.getExtraProperties() != null) {
if (model.getExtraProperties().containsKey("license") && model.getExtraProperties().get("license") != null) {
Object license = model.getExtraProperties().get("license");
HashMap<String, String> licenseMap = this.jsonHandlingService.mapFromJson(this.jsonHandlingService.toJson(license));
ReferenceEntity referenceEntity = new ReferenceEntity();
String licensePid;
if (licenseMap.containsKey("pid") && licenseMap.get("pid") != null) {
licensePid = licenseMap.get("pid");
referenceEntity.setReference(licensePid);
}
else
continue;
boolean licenseExists = collectedLicenses.contains(licensePid);
if (!licenseExists) {
DefinitionEntity definitionEntity = new DefinitionEntity();
List<FieldEntity> fields = new ArrayList<>();
if (licenseMap.containsKey("name")) {
referenceEntity.setLabel(licenseMap.get("name"));
}
if (licenseMap.containsKey("pid")) {
if (!this.conventionService.isNullOrEmpty(licensePid)){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("pid");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licensePid);
fields.add(fieldEntity);
}
}
if (licenseMap.containsKey("tag")) {
if (!this.conventionService.isNullOrEmpty(licenseMap.get("tag"))){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("tag");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licenseMap.get("tag"));
fields.add(fieldEntity);
}
}
if (licenseMap.containsKey("uri")) {
referenceEntity.setReference(licenseMap.get("uri"));
if (!this.conventionService.isNullOrEmpty(licenseMap.get("uri"))){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("uri");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licenseMap.get("uri"));
fields.add(fieldEntity);
}
}
if (licenseMap.containsKey("abbreviation")) {
referenceEntity.setAbbreviation(licenseMap.get("abbreviation"));
}
if (licenseMap.containsKey("created") && licenseMap.get("created") != null) {
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("created")));
} else {
referenceEntity.setCreatedAt(item.getCreated() == null ? Instant.now() : item.getCreated().toInstant());
}
if (licenseMap.containsKey("modified") && licenseMap.get("modified") != null) {
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("modified")));
} else {
referenceEntity.setUpdatedAt(item.getModified() == null ? Instant.now() : item.getModified().toInstant());
}
referenceEntity.setId(UUID.randomUUID());
referenceEntity.setSourceType(ReferenceSourceType.External);
referenceEntity.setSource(this.environment.getProperty("migration.default-license-source", "null"));
referenceEntity.setType(ReferenceType.Licenses);
referenceEntity.setIsActive(IsActive.Active);
definitionEntity.setFields(fields);
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
collectedLicenses.add(licensePid); if (systemFieldEntities == null) {
licenseIdByName.put(licensePid, referenceEntity.getId()); logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
logger.debug("License '{}' migrated", licensePid); throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
} }
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Grant)).findFirst().orElse(null);
if (systemField == null) {
logger.warn("Migration failed Grant field not found " + item.getId());
throw new MyApplicationException("Migration failed Grant field not found " + item.getId());
}
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(item.getId());
data.setReferenceId(grant.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
if (grant.getFunder() != null){
DmpReferenceEntity dmpReferenceEntity = new DmpReferenceEntity(); SystemFieldEntity funderSystemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Funder)).findFirst().orElse(null);
dmpReferenceEntity.setId(UUID.randomUUID()); if (funderSystemField == null) {
dmpReferenceEntity.setDmpId(item.getId()); logger.warn("Migration failed Funder field not found " + item.getId());
dmpReferenceEntity.setReferenceId(licenseIdByName.get(licensePid)); throw new MyApplicationException("Migration failed Funder field not found " + item.getId());
dmpReferenceEntity.setCreatedAt(Instant.now()); }
dmpReferenceEntity.setUpdatedAt(Instant.now()); DmpReferenceEntity founder = new DmpReferenceEntity();
dmpReferenceEntity.setIsActive(IsActive.Active); founder.setId(UUID.randomUUID());
founder.setDmpId(item.getId());
founder.setReferenceId(grant.getFunder().getId());
founder.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
founder.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
founder.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(funderSystemField)));
founder.setIsActive(IsActive.Active);
this.entityManager.persist(founder);
}
this.entityManager.flush();
}
if (!licenseExists) private Map<String, UUID> migrateLicense(DMP item, DataManagementPlan model, Map<String, UUID> licenseIdByName, List<SystemFieldEntity> systemFieldEntities) throws JsonProcessingException {
entityManager.persist(referenceEntity); if (model.getExtraProperties() != null) {
entityManager.persist(dmpReferenceEntity); if (model.getExtraProperties().containsKey("license") && model.getExtraProperties().get("license") != null) {
if (systemFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.License)).findFirst().orElse(null);
if (systemField == null) {
logger.warn("Migration failed License field not found " + item.getId());
throw new MyApplicationException("Migration failed License field not found " + item.getId());
}
Object license = model.getExtraProperties().get("license");
HashMap<String, String> licenseMap = this.jsonHandlingService.mapFromJson(this.jsonHandlingService.toJson(license));
ReferenceEntity referenceEntity = new ReferenceEntity();
String licensePid;
if (licenseMap.containsKey("pid") && licenseMap.get("pid") != null) {
licensePid = licenseMap.get("pid");
referenceEntity.setReference(licensePid);
}
else return licenseIdByName;
boolean licenseExists = licenseIdByName.containsKey(licensePid);
if (!licenseExists) {
DefinitionEntity definitionEntity = new DefinitionEntity();
List<FieldEntity> fields = new ArrayList<>();
if (licenseMap.containsKey("name")) {
referenceEntity.setLabel(licenseMap.get("name"));
}
if (licenseMap.containsKey("pid")) {
if (!this.conventionService.isNullOrEmpty(licensePid)){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("pid");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licensePid);
fields.add(fieldEntity);
} }
} }
this.entityManager.flush(); if (licenseMap.containsKey("tag")) {
if (!this.conventionService.isNullOrEmpty(licenseMap.get("tag"))){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("tag");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licenseMap.get("tag"));
fields.add(fieldEntity);
}
}
if (licenseMap.containsKey("uri")) {
referenceEntity.setReference(licenseMap.get("uri"));
if (!this.conventionService.isNullOrEmpty(licenseMap.get("uri"))){
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("uri");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(licenseMap.get("uri"));
fields.add(fieldEntity);
}
}
if (licenseMap.containsKey("abbreviation")) {
referenceEntity.setAbbreviation(licenseMap.get("abbreviation"));
}
if (licenseMap.containsKey("created") && licenseMap.get("created") != null) {
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("created")));
} else {
referenceEntity.setCreatedAt(item.getCreated() == null ? Instant.now() : item.getCreated().toInstant());
}
if (licenseMap.containsKey("modified") && licenseMap.get("modified") != null) {
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("modified")));
} else {
referenceEntity.setUpdatedAt(item.getModified() == null ? Instant.now() : item.getModified().toInstant());
}
referenceEntity.setId(UUID.randomUUID());
referenceEntity.setSourceType(ReferenceSourceType.External);
referenceEntity.setSource(this.environment.getProperty("migration.default-license-source", "null"));
referenceEntity.setType(ReferenceType.Licenses);
referenceEntity.setIsActive(IsActive.Active);
definitionEntity.setFields(fields);
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
licenseIdByName.put(licensePid, referenceEntity.getId());
logger.debug("License '{}' migrated", licensePid);
} }
page++; DmpReferenceEntity dmpReferenceEntity = new DmpReferenceEntity();
dmpReferenceEntity.setId(UUID.randomUUID());
dmpReferenceEntity.setDmpId(item.getId());
dmpReferenceEntity.setReferenceId(licenseIdByName.get(licensePid));
dmpReferenceEntity.setCreatedAt(Instant.now());
dmpReferenceEntity.setUpdatedAt(Instant.now());
dmpReferenceEntity.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
dmpReferenceEntity.setIsActive(IsActive.Active);
if (!licenseExists) entityManager.persist(referenceEntity);
entityManager.persist(dmpReferenceEntity);
} }
} while (items != null && !items.isEmpty() && !TestMode); }
logger.info("Dmp licenses migration finished"); this.entityManager.flush();
return licenseIdByName;
}
private DmpReferenceDataEntity buildDmpReferenceDataEntity(SystemFieldEntity systemField){
DmpReferenceDataEntity data = new DmpReferenceDataEntity();
data.setBlueprintFieldId(systemField.getId());
return data;
} }
} }

154
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceMigrationService.java
View File

@ -21,24 +21,15 @@ public class ReferenceMigrationService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class)); private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class));
private final DatabaseRepository databaseRepository; private final DatabaseRepository databaseRepository;
private final QueryFactory queryFactory;
private static final int PageSize = 500; private static final int PageSize = 500;
private static final boolean TestMode = false; private static final boolean TestMode = false;
private final EntityManager entityManager; private final EntityManager entityManager;
public ReferenceMigrationService(DatabaseRepository databaseRepository, QueryFactory queryFactory, EntityManager entityManager) { public ReferenceMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) {
this.databaseRepository = databaseRepository; this.databaseRepository = databaseRepository;
this.queryFactory = queryFactory;
this.entityManager = entityManager; this.entityManager = entityManager;
} }
public void migrateDmpReferences() {
migrateDmpOrganizations();
migrateDmpResearchers();
migrateDmpProjects();
migrateDmpGrantsAndFounders();
}
public void migrateDatasetReferences() { public void migrateDatasetReferences() {
migrateDatasetDataRepositories(); migrateDatasetDataRepositories();
migrateDatasetExternalDatasets(); migrateDatasetExternalDatasets();
@ -46,149 +37,6 @@ public class ReferenceMigrationService {
migrateDatasetServices(); migrateDatasetServices();
} }
public void migrateDmpOrganizations() {
OrganisationDao organisationDao = databaseRepository.getOrganisationDao();
long total = organisationDao.asQueryable().count();
logger.debug("Migrate Dmp Organisation (from Organization) Total : " + total);
int page = 0;
List<Organisation> items;
do {
items = organisationDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dmp Organisation " + page * PageSize + " of " + total);
for (Organisation item : items) {
// entityManager.detach(item);
for (DMP dmp : item.getDmps()) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(dmp.getId());
data.setReferenceId(item.getId());
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDmpResearchers() {
ResearcherDao researcherDao = databaseRepository.getResearcherDao();
long total = researcherDao.asQueryable().count();
logger.debug("Migrate Dmp Researcher (from Researcher) Total : " + total);
int page = 0;
List<Researcher> items;
do {
items = researcherDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dmp Researcher " + page * PageSize + " of " + total);
for (Researcher item : items) {
// entityManager.detach(item);
for (DMP dmp : item.getdMPs()) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(dmp.getId());
data.setReferenceId(item.getId());
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDmpProjects() {
ProjectDao projectDao = databaseRepository.getProjectDao();
long total = projectDao.asQueryable().count();
logger.debug("Migrate Dmp Project (from Project) Total : " + total);
int page = 0;
List<Project> items;
do {
items = projectDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dmp Project " + page * PageSize + " of " + total);
for (Project item : items) {
// entityManager.detach(item);
for (DMP dmp : item.getDmps()) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(dmp.getId());
data.setReferenceId(item.getId());
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDmpGrantsAndFounders() {
GrantDao grantDao = databaseRepository.getGrantDao();
long total = grantDao.asQueryable().count();
logger.debug("Migrate Dmp Grants (from Grants) Total : " + total);
int page = 0;
List<Grant> items;
do {
items = grantDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dmp Grants " + page * PageSize + " of " + total);
for (Grant item : items) {
// entityManager.detach(item);
for (DMP dmp : item.getDmps()) {
DmpReferenceEntity data = new DmpReferenceEntity();
data.setId(UUID.randomUUID());
data.setDmpId(dmp.getId());
data.setReferenceId(item.getId());
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
if (item.getFunder() != null){
DmpReferenceEntity founder = new DmpReferenceEntity();
founder.setId(UUID.randomUUID());
founder.setDmpId(dmp.getId());
founder.setReferenceId(item.getFunder().getId());
founder.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
founder.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
founder.setIsActive(IsActive.Active);
this.entityManager.persist(founder);
}
}
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty());
}
public void migrateDatasetDataRepositories() { public void migrateDatasetDataRepositories() {
DatasetDataRepositoryDao datasetDataRepositoryDao = databaseRepository.getDatasetDataRepositoryDao(); DatasetDataRepositoryDao datasetDataRepositoryDao = databaseRepository.getDatasetDataRepositoryDao();
long total = datasetDataRepositoryDao.asQueryable().count(); long total = datasetDataRepositoryDao.asQueryable().count();

18
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -87,7 +87,6 @@ public class MigrationController {
this.serviceMigrationService.migrate(); this.serviceMigrationService.migrate();
this.dmpMigrationService.migrate(); this.dmpMigrationService.migrate();
this.dmpMigrationService.migrateDmpLicenses();
this.dmpDatasetProfileMigrationService.migrate(); this.dmpDatasetProfileMigrationService.migrate();
this.datasetMigrationService.migrate(); this.datasetMigrationService.migrate();
this.tagMigrationService.migrate(); this.tagMigrationService.migrate();
@ -95,7 +94,6 @@ public class MigrationController {
this.dmpUserMigrationService.migrate(); this.dmpUserMigrationService.migrate();
this.referenceMigrationService.migrateDatasetReferences(); this.referenceMigrationService.migrateDatasetReferences();
this.referenceMigrationService.migrateDmpReferences();
this.userContactInfoMigrationService.migrate(); this.userContactInfoMigrationService.migrate();
this.userMigrationService.migrate(); this.userMigrationService.migrate();
@ -121,18 +119,11 @@ public class MigrationController {
@GetMapping("dmps") @GetMapping("dmps")
@Transactional @Transactional
public boolean migrateDmps() throws JsonProcessingException, NoSuchFieldException, IllegalAccessException { public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
this.dmpMigrationService.migrate(); this.dmpMigrationService.migrate();
return true; return true;
} }
@GetMapping("dmps/licenses")
@Transactional
public boolean migrateDmpLicenses() throws JsonProcessingException, NoSuchFieldException, IllegalAccessException {
this.dmpMigrationService.migrateDmpLicenses();
return true;
}
@GetMapping("datasets") @GetMapping("datasets")
@Transactional @Transactional
public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException { public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
@ -169,13 +160,6 @@ public class MigrationController {
return true; return true;
} }
@GetMapping("dmp-references")
@Transactional
public boolean migrateDmpReferences() {
this.referenceMigrationService.migrateDmpReferences();
return true;
}
@GetMapping("dmp-users") @GetMapping("dmp-users")
@Transactional @Transactional
public boolean migrateDmpUsers() { public boolean migrateDmpUsers() {