reference migration changes
This commit is contained in:
parent
f49f6d1c04
commit
d5292f5cec
|
@ -2,7 +2,7 @@ deposit:
|
|||
sources:
|
||||
- url: http://localhost:8082
|
||||
repositoryId: zenodo
|
||||
issuer-url: ${ZENODO_ISSUER_URI:IDP_APIKEY_ISSUER_URI}
|
||||
issuer-url: ${ZENODO_ISSUER_URI:IDP_APIKEY_ISSUER_URI}/protocol/openid-connect/token
|
||||
client-id: ${ZENODO_DEPOSIT_CLIENT_ID:}
|
||||
client-secret: ${ZENODO_DEPOSIT_CLIENT_SECRET:}
|
||||
scope: ${ZENODO_DEPOSIT_SCOPE:}
|
|
@ -1,29 +1,53 @@
|
|||
package eu.old.eudat.data.entities;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonValue;
|
||||
import eu.eudat.commons.enums.EnumUtils;
|
||||
import eu.eudat.data.converters.enums.DatabaseEnum;
|
||||
import eu.eudat.data.converters.enums.DatabaseEnumConverter;
|
||||
import eu.old.eudat.data.converters.DateToUTCConverter;
|
||||
import eu.old.eudat.data.entities.helpers.EntityBinder;
|
||||
import eu.old.eudat.queryable.queryableentity.DataEntity;
|
||||
|
||||
import jakarta.persistence.*;
|
||||
import java.util.Arrays;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
import java.util.UUID;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Entity
|
||||
@Table(name = "\"EntityDoi\"")
|
||||
public class EntityDoi implements DataEntity<EntityDoi, UUID> {
|
||||
public enum EntityType {
|
||||
DMP
|
||||
public enum EntityType implements DatabaseEnum<Short> {
|
||||
|
||||
DMP((short) 0);
|
||||
|
||||
private final Short value;
|
||||
|
||||
EntityType(Short value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
@Override
|
||||
@JsonValue
|
||||
public Short getValue() {
|
||||
return value;
|
||||
}
|
||||
|
||||
private static final Map<Short, EntityType> map = EnumUtils.getEnumValueMap(EntityType.class);
|
||||
|
||||
public static EntityType of(Short i) {
|
||||
return map.get(i);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@Id
|
||||
@Column(name = "id", updatable = false, nullable = false, columnDefinition = "BINARY(16)")
|
||||
private UUID id;
|
||||
|
||||
@Enumerated(EnumType.STRING)
|
||||
@Column(name = "entity_type", nullable = false)
|
||||
@Convert(converter = EntityTypeConverter.class)
|
||||
private EntityType entityType;
|
||||
|
||||
@Column(name = "repository_id", nullable = false)
|
||||
|
|
|
@ -0,0 +1,14 @@
|
|||
package eu.old.eudat.data.entities;
|
||||
|
||||
import eu.eudat.data.converters.enums.DatabaseEnumConverter;
|
||||
import jakarta.persistence.Converter;
|
||||
|
||||
@Converter
|
||||
public class EntityTypeConverter extends DatabaseEnumConverter<EntityDoi.EntityType, Short> {
|
||||
|
||||
@Override
|
||||
protected EntityDoi.EntityType of(Short i) {
|
||||
return EntityDoi.EntityType.of(i);
|
||||
}
|
||||
|
||||
}
|
|
@ -84,7 +84,7 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
|
|||
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().should(QueryBuilders.termsQuery("datasets.id.keyword", ids));
|
||||
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery( "datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
|
||||
searchSourceBuilder.query(nestedQueryBuilder);
|
||||
searchSourceBuilder.size(100000);
|
||||
searchSourceBuilder.size(10000);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
|
||||
return Arrays.stream(response.getHits().getHits())
|
||||
|
|
|
@ -1,38 +1,44 @@
|
|||
package eu.old.eudat.migration;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.fasterxml.jackson.databind.type.MapType;
|
||||
import com.fasterxml.jackson.databind.type.TypeFactory;
|
||||
import eu.eudat.commons.JsonHandlingService;
|
||||
import eu.eudat.commons.XmlHandlingService;
|
||||
import eu.eudat.commons.enums.*;
|
||||
import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
|
||||
import eu.eudat.commons.types.dmp.DmpContactEntity;
|
||||
import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
|
||||
import eu.eudat.commons.types.dmpblueprint.SectionEntity;
|
||||
import eu.eudat.commons.types.dmpblueprint.SystemFieldEntity;
|
||||
import eu.eudat.commons.types.dmpreference.DmpReferenceDataEntity;
|
||||
import eu.eudat.commons.types.reference.DefinitionEntity;
|
||||
import eu.eudat.commons.types.reference.FieldEntity;
|
||||
import eu.eudat.convention.ConventionService;
|
||||
import eu.eudat.data.DmpBlueprintEntity;
|
||||
import eu.eudat.data.DmpEntity;
|
||||
import eu.eudat.data.DmpReferenceEntity;
|
||||
import eu.eudat.data.ReferenceEntity;
|
||||
import eu.eudat.query.ReferenceQuery;
|
||||
import eu.eudat.model.DmpBlueprint;
|
||||
import eu.eudat.query.DmpBlueprintQuery;
|
||||
import eu.old.eudat.data.dao.entities.DMPDao;
|
||||
import eu.old.eudat.data.entities.DMP;
|
||||
import eu.old.eudat.data.entities.*;
|
||||
import eu.old.eudat.logic.services.operations.DatabaseRepository;
|
||||
import eu.old.eudat.models.data.dmp.DataManagementPlan;
|
||||
import gr.cite.tools.data.query.QueryFactory;
|
||||
import gr.cite.tools.exception.MyApplicationException;
|
||||
import gr.cite.tools.fieldset.BaseFieldSet;
|
||||
import gr.cite.tools.logging.LoggerService;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.xml.bind.JAXBException;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.core.env.Environment;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.xml.sax.SAXException;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import javax.xml.parsers.ParserConfigurationException;
|
||||
import java.io.IOException;
|
||||
import java.time.Instant;
|
||||
import java.util.*;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
@Service
|
||||
public class DmpMigrationService {
|
||||
|
@ -44,30 +50,33 @@ public class DmpMigrationService {
|
|||
private final ConventionService conventionService;
|
||||
private final XmlHandlingService xmlHandlingService;
|
||||
private final Environment environment;
|
||||
private final QueryFactory queryFactory;
|
||||
|
||||
private static final int PageSize = 500;
|
||||
private static final boolean TestMode = false;
|
||||
|
||||
public DmpMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, EntityManager entityManager, ConventionService conventionService, XmlHandlingService xmlHandlingService, Environment environment) {
|
||||
public DmpMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, EntityManager entityManager, ConventionService conventionService, XmlHandlingService xmlHandlingService, Environment environment, QueryFactory queryFactory) {
|
||||
this.databaseRepository = databaseRepository;
|
||||
this.jsonHandlingService = jsonHandlingService;
|
||||
this.entityManager = entityManager;
|
||||
this.conventionService = conventionService;
|
||||
this.xmlHandlingService = xmlHandlingService;
|
||||
this.environment = environment;
|
||||
this.queryFactory = queryFactory;
|
||||
}
|
||||
|
||||
public void migrate() throws JsonProcessingException, NoSuchFieldException, IllegalAccessException {
|
||||
public void migrate() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
|
||||
DMPDao dmpDao = databaseRepository.getDmpDao();
|
||||
long total = dmpDao.asQueryable().count();
|
||||
logger.debug("Migrate Dmp Total : " + total);
|
||||
int page = 0;
|
||||
|
||||
Map<String, UUID> licenseIdByName = new HashMap<>();
|
||||
List<DMP> items;
|
||||
do {
|
||||
items = dmpDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
|
||||
List<DMP> finalItems = items;
|
||||
List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("groupId").in(finalItems.stream().map(DMP::getGroupId).distinct())).toList();
|
||||
List<DMP> groupDmps = dmpDao.asQueryable().where((builder, root) -> root.get("groupId").in(finalItems.stream().map(DMP::getGroupId).distinct().collect(Collectors.toList()))).toList();
|
||||
Map<UUID, List<DMP>> groupDmpMap = new HashMap<>();
|
||||
for (DMP dmp: groupDmps) {
|
||||
if (!groupDmpMap.containsKey(dmp.getGroupId())) groupDmpMap.put(dmp.getGroupId(), new ArrayList<>());
|
||||
|
@ -75,8 +84,15 @@ public class DmpMigrationService {
|
|||
}
|
||||
|
||||
if (items != null && !items.isEmpty()) {
|
||||
|
||||
logger.debug("Migrate Dmp " + page * PageSize + " of " + total);
|
||||
|
||||
List<DmpBlueprintEntity> dmpBlueprintEntities = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().filter(x-> x.getProfile() != null).map(x-> x.getProfile().getId()).distinct().collect(Collectors.toList())).collectAs(new BaseFieldSet().ensure(DmpBlueprint._definition).ensure(DmpBlueprint._id));
|
||||
Map<UUID, List<SystemFieldEntity>> systemFieldsByDmpBlueprintId = new HashMap<>();
|
||||
for (DmpBlueprintEntity dmpBlueprintEntity : dmpBlueprintEntities) {
|
||||
eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition =this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition());
|
||||
List<SystemFieldEntity> systemFieldEntities = definition.getSections().stream().map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.System)).map(x-> (SystemFieldEntity)x).collect(Collectors.toList());
|
||||
systemFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , systemFieldEntities);
|
||||
}
|
||||
for (DMP item : items) {
|
||||
//entityManager.detach(item);
|
||||
|
||||
|
@ -122,14 +138,14 @@ public class DmpMigrationService {
|
|||
data.setAccessType((boolean) model.getExtraProperties().get("visible") ? DmpAccessType.Public : DmpAccessType.Restricted);
|
||||
if (model.getExtraProperties().containsKey("contact") && model.getExtraProperties().get("contact") != null) {
|
||||
DmpContactEntity contactEntity = new DmpContactEntity();
|
||||
contactEntity.setUserId((String) model.getExtraProperties().get("contact"));
|
||||
contactEntity.setUserId(UUID.fromString((String)model.getExtraProperties().get("contact")));
|
||||
dmpProperties.getContacts().add(contactEntity);
|
||||
}
|
||||
}
|
||||
if (model.getProperties() != null) {
|
||||
model.getProperties().forEach((key,val) -> {
|
||||
DmpBlueprintValueEntity valueEntity = new DmpBlueprintValueEntity();
|
||||
valueEntity.setFieldId(key);
|
||||
valueEntity.setFieldId(UUID.fromString(key));
|
||||
valueEntity.setValue((String) val);
|
||||
dmpProperties.getDmpBlueprintValues().add(valueEntity);
|
||||
});
|
||||
|
@ -158,124 +174,247 @@ public class DmpMigrationService {
|
|||
|
||||
this.entityManager.persist(data);
|
||||
this.entityManager.flush();
|
||||
|
||||
licenseIdByName = this.migrateLicense(item, model, licenseIdByName, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
|
||||
this.migrateOrganizations(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
|
||||
this.migrateResearchers(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
|
||||
this.migrateProjects(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
|
||||
this.migrateGrantAndFunder(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
|
||||
}
|
||||
|
||||
page++;
|
||||
}
|
||||
} while (items != null && !items.isEmpty() && !TestMode);
|
||||
}
|
||||
|
||||
private void migrateOrganizations(DMP item, List<SystemFieldEntity> systemFieldEntities){
|
||||
Set<Organisation> organisations = item.getOrganisations();
|
||||
if (organisations == null || organisations.isEmpty()) return;
|
||||
|
||||
public void migrateDmpLicenses() throws JsonProcessingException {
|
||||
DMPDao dmpDao = databaseRepository.getDmpDao();
|
||||
long total = dmpDao.asQueryable().count();
|
||||
logger.debug("Migrate Licenses for Dmp Total : " + total);
|
||||
int page = 0;
|
||||
if (systemFieldEntities == null) {
|
||||
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
}
|
||||
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Organizations)).findFirst().orElse(null);
|
||||
if (systemField == null) {
|
||||
logger.warn("Migration failed Organizations field not found " + item.getId());
|
||||
throw new MyApplicationException("Migration failed Organizations field not found " + item.getId());
|
||||
}
|
||||
|
||||
for (Organisation organisation : organisations) {
|
||||
DmpReferenceEntity data = new DmpReferenceEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
data.setDmpId(item.getId());
|
||||
data.setReferenceId(organisation.getId());
|
||||
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
|
||||
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
|
||||
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
|
||||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
}
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
Set<String> collectedLicenses = new HashSet<>();
|
||||
Map<String, UUID> licenseIdByName = new HashMap<>();
|
||||
List<DMP> items;
|
||||
do {
|
||||
items = dmpDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
|
||||
private void migrateProjects(DMP item, List<SystemFieldEntity> systemFieldEntities){
|
||||
Project project = item.getProject();
|
||||
if (project == null) return;
|
||||
if (systemFieldEntities == null) {
|
||||
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
}
|
||||
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Project)).findFirst().orElse(null);
|
||||
if (systemField == null) {
|
||||
logger.warn("Migration failed Project field not found " + item.getId());
|
||||
throw new MyApplicationException("Migration failed Project field not found " + item.getId());
|
||||
}
|
||||
|
||||
DmpReferenceEntity data = new DmpReferenceEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
data.setDmpId(item.getId());
|
||||
data.setReferenceId(project.getId());
|
||||
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
|
||||
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
|
||||
data.setIsActive(IsActive.Active);
|
||||
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
|
||||
this.entityManager.persist(data);
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
if (items != null && !items.isEmpty()) {
|
||||
logger.debug("Migrate Licenses for Dmp " + page * PageSize + " of " + total);
|
||||
private void migrateResearchers(DMP item, List<SystemFieldEntity> systemFieldEntities){
|
||||
Set<Researcher> researchers = item.getResearchers();
|
||||
if (researchers == null || researchers.isEmpty()) return;
|
||||
if (systemFieldEntities == null) {
|
||||
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
}
|
||||
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Researchers)).findFirst().orElse(null);
|
||||
if (systemField == null) {
|
||||
logger.warn("Migration failed Researchers field not found " + item.getId());
|
||||
throw new MyApplicationException("Migration failed Researchers field not found " + item.getId());
|
||||
}
|
||||
|
||||
for (Researcher researcher : researchers) {
|
||||
DmpReferenceEntity data = new DmpReferenceEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
data.setDmpId(item.getId());
|
||||
data.setReferenceId(researcher.getId());
|
||||
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
|
||||
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
|
||||
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
|
||||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
}
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
for (DMP item : items) {
|
||||
DataManagementPlan model = new DataManagementPlan();
|
||||
model.fromDataModel(item);
|
||||
if (model.getExtraProperties() != null) {
|
||||
if (model.getExtraProperties().containsKey("license") && model.getExtraProperties().get("license") != null) {
|
||||
Object license = model.getExtraProperties().get("license");
|
||||
HashMap<String, String> licenseMap = this.jsonHandlingService.mapFromJson(this.jsonHandlingService.toJson(license));
|
||||
ReferenceEntity referenceEntity = new ReferenceEntity();
|
||||
String licensePid;
|
||||
if (licenseMap.containsKey("pid") && licenseMap.get("pid") != null) {
|
||||
licensePid = licenseMap.get("pid");
|
||||
referenceEntity.setReference(licensePid);
|
||||
}
|
||||
else
|
||||
continue;
|
||||
boolean licenseExists = collectedLicenses.contains(licensePid);
|
||||
if (!licenseExists) {
|
||||
DefinitionEntity definitionEntity = new DefinitionEntity();
|
||||
List<FieldEntity> fields = new ArrayList<>();
|
||||
if (licenseMap.containsKey("name")) {
|
||||
referenceEntity.setLabel(licenseMap.get("name"));
|
||||
}
|
||||
if (licenseMap.containsKey("pid")) {
|
||||
if (!this.conventionService.isNullOrEmpty(licensePid)){
|
||||
FieldEntity fieldEntity = new FieldEntity();
|
||||
fieldEntity.setCode("pid");
|
||||
fieldEntity.setDataType(ReferenceFieldDataType.Text);
|
||||
fieldEntity.setValue(licensePid);
|
||||
fields.add(fieldEntity);
|
||||
}
|
||||
}
|
||||
if (licenseMap.containsKey("tag")) {
|
||||
if (!this.conventionService.isNullOrEmpty(licenseMap.get("tag"))){
|
||||
FieldEntity fieldEntity = new FieldEntity();
|
||||
fieldEntity.setCode("tag");
|
||||
fieldEntity.setDataType(ReferenceFieldDataType.Text);
|
||||
fieldEntity.setValue(licenseMap.get("tag"));
|
||||
fields.add(fieldEntity);
|
||||
}
|
||||
}
|
||||
if (licenseMap.containsKey("uri")) {
|
||||
referenceEntity.setReference(licenseMap.get("uri"));
|
||||
if (!this.conventionService.isNullOrEmpty(licenseMap.get("uri"))){
|
||||
FieldEntity fieldEntity = new FieldEntity();
|
||||
fieldEntity.setCode("uri");
|
||||
fieldEntity.setDataType(ReferenceFieldDataType.Text);
|
||||
fieldEntity.setValue(licenseMap.get("uri"));
|
||||
fields.add(fieldEntity);
|
||||
}
|
||||
}
|
||||
if (licenseMap.containsKey("abbreviation")) {
|
||||
referenceEntity.setAbbreviation(licenseMap.get("abbreviation"));
|
||||
}
|
||||
if (licenseMap.containsKey("created") && licenseMap.get("created") != null) {
|
||||
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("created")));
|
||||
} else {
|
||||
referenceEntity.setCreatedAt(item.getCreated() == null ? Instant.now() : item.getCreated().toInstant());
|
||||
}
|
||||
if (licenseMap.containsKey("modified") && licenseMap.get("modified") != null) {
|
||||
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("modified")));
|
||||
} else {
|
||||
referenceEntity.setUpdatedAt(item.getModified() == null ? Instant.now() : item.getModified().toInstant());
|
||||
}
|
||||
referenceEntity.setId(UUID.randomUUID());
|
||||
referenceEntity.setSourceType(ReferenceSourceType.External);
|
||||
referenceEntity.setSource(this.environment.getProperty("migration.default-license-source", "null"));
|
||||
referenceEntity.setType(ReferenceType.Licenses);
|
||||
referenceEntity.setIsActive(IsActive.Active);
|
||||
definitionEntity.setFields(fields);
|
||||
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
|
||||
private void migrateGrantAndFunder(DMP item, List<SystemFieldEntity> systemFieldEntities){
|
||||
Grant grant = item.getGrant();
|
||||
if (grant == null) return;
|
||||
|
||||
collectedLicenses.add(licensePid);
|
||||
licenseIdByName.put(licensePid, referenceEntity.getId());
|
||||
logger.debug("License '{}' migrated", licensePid);
|
||||
}
|
||||
if (systemFieldEntities == null) {
|
||||
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
}
|
||||
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Grant)).findFirst().orElse(null);
|
||||
if (systemField == null) {
|
||||
logger.warn("Migration failed Grant field not found " + item.getId());
|
||||
throw new MyApplicationException("Migration failed Grant field not found " + item.getId());
|
||||
}
|
||||
|
||||
DmpReferenceEntity data = new DmpReferenceEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
data.setDmpId(item.getId());
|
||||
data.setReferenceId(grant.getId());
|
||||
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
|
||||
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
|
||||
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
|
||||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
if (grant.getFunder() != null){
|
||||
|
||||
DmpReferenceEntity dmpReferenceEntity = new DmpReferenceEntity();
|
||||
dmpReferenceEntity.setId(UUID.randomUUID());
|
||||
dmpReferenceEntity.setDmpId(item.getId());
|
||||
dmpReferenceEntity.setReferenceId(licenseIdByName.get(licensePid));
|
||||
dmpReferenceEntity.setCreatedAt(Instant.now());
|
||||
dmpReferenceEntity.setUpdatedAt(Instant.now());
|
||||
dmpReferenceEntity.setIsActive(IsActive.Active);
|
||||
SystemFieldEntity funderSystemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Funder)).findFirst().orElse(null);
|
||||
if (funderSystemField == null) {
|
||||
logger.warn("Migration failed Funder field not found " + item.getId());
|
||||
throw new MyApplicationException("Migration failed Funder field not found " + item.getId());
|
||||
}
|
||||
DmpReferenceEntity founder = new DmpReferenceEntity();
|
||||
founder.setId(UUID.randomUUID());
|
||||
founder.setDmpId(item.getId());
|
||||
founder.setReferenceId(grant.getFunder().getId());
|
||||
founder.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
|
||||
founder.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
|
||||
founder.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(funderSystemField)));
|
||||
founder.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(founder);
|
||||
}
|
||||
this.entityManager.flush();
|
||||
}
|
||||
|
||||
if (!licenseExists)
|
||||
entityManager.persist(referenceEntity);
|
||||
entityManager.persist(dmpReferenceEntity);
|
||||
private Map<String, UUID> migrateLicense(DMP item, DataManagementPlan model, Map<String, UUID> licenseIdByName, List<SystemFieldEntity> systemFieldEntities) throws JsonProcessingException {
|
||||
if (model.getExtraProperties() != null) {
|
||||
if (model.getExtraProperties().containsKey("license") && model.getExtraProperties().get("license") != null) {
|
||||
if (systemFieldEntities == null) {
|
||||
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
|
||||
}
|
||||
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.License)).findFirst().orElse(null);
|
||||
if (systemField == null) {
|
||||
logger.warn("Migration failed License field not found " + item.getId());
|
||||
throw new MyApplicationException("Migration failed License field not found " + item.getId());
|
||||
}
|
||||
|
||||
Object license = model.getExtraProperties().get("license");
|
||||
HashMap<String, String> licenseMap = this.jsonHandlingService.mapFromJson(this.jsonHandlingService.toJson(license));
|
||||
ReferenceEntity referenceEntity = new ReferenceEntity();
|
||||
String licensePid;
|
||||
if (licenseMap.containsKey("pid") && licenseMap.get("pid") != null) {
|
||||
licensePid = licenseMap.get("pid");
|
||||
referenceEntity.setReference(licensePid);
|
||||
}
|
||||
else return licenseIdByName;
|
||||
|
||||
boolean licenseExists = licenseIdByName.containsKey(licensePid);
|
||||
if (!licenseExists) {
|
||||
DefinitionEntity definitionEntity = new DefinitionEntity();
|
||||
List<FieldEntity> fields = new ArrayList<>();
|
||||
if (licenseMap.containsKey("name")) {
|
||||
referenceEntity.setLabel(licenseMap.get("name"));
|
||||
}
|
||||
if (licenseMap.containsKey("pid")) {
|
||||
if (!this.conventionService.isNullOrEmpty(licensePid)){
|
||||
FieldEntity fieldEntity = new FieldEntity();
|
||||
fieldEntity.setCode("pid");
|
||||
fieldEntity.setDataType(ReferenceFieldDataType.Text);
|
||||
fieldEntity.setValue(licensePid);
|
||||
fields.add(fieldEntity);
|
||||
}
|
||||
}
|
||||
this.entityManager.flush();
|
||||
if (licenseMap.containsKey("tag")) {
|
||||
if (!this.conventionService.isNullOrEmpty(licenseMap.get("tag"))){
|
||||
FieldEntity fieldEntity = new FieldEntity();
|
||||
fieldEntity.setCode("tag");
|
||||
fieldEntity.setDataType(ReferenceFieldDataType.Text);
|
||||
fieldEntity.setValue(licenseMap.get("tag"));
|
||||
fields.add(fieldEntity);
|
||||
}
|
||||
}
|
||||
if (licenseMap.containsKey("uri")) {
|
||||
referenceEntity.setReference(licenseMap.get("uri"));
|
||||
if (!this.conventionService.isNullOrEmpty(licenseMap.get("uri"))){
|
||||
FieldEntity fieldEntity = new FieldEntity();
|
||||
fieldEntity.setCode("uri");
|
||||
fieldEntity.setDataType(ReferenceFieldDataType.Text);
|
||||
fieldEntity.setValue(licenseMap.get("uri"));
|
||||
fields.add(fieldEntity);
|
||||
}
|
||||
}
|
||||
if (licenseMap.containsKey("abbreviation")) {
|
||||
referenceEntity.setAbbreviation(licenseMap.get("abbreviation"));
|
||||
}
|
||||
if (licenseMap.containsKey("created") && licenseMap.get("created") != null) {
|
||||
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("created")));
|
||||
} else {
|
||||
referenceEntity.setCreatedAt(item.getCreated() == null ? Instant.now() : item.getCreated().toInstant());
|
||||
}
|
||||
if (licenseMap.containsKey("modified") && licenseMap.get("modified") != null) {
|
||||
referenceEntity.setUpdatedAt(Instant.parse(licenseMap.get("modified")));
|
||||
} else {
|
||||
referenceEntity.setUpdatedAt(item.getModified() == null ? Instant.now() : item.getModified().toInstant());
|
||||
}
|
||||
referenceEntity.setId(UUID.randomUUID());
|
||||
referenceEntity.setSourceType(ReferenceSourceType.External);
|
||||
referenceEntity.setSource(this.environment.getProperty("migration.default-license-source", "null"));
|
||||
referenceEntity.setType(ReferenceType.Licenses);
|
||||
referenceEntity.setIsActive(IsActive.Active);
|
||||
definitionEntity.setFields(fields);
|
||||
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
|
||||
|
||||
licenseIdByName.put(licensePid, referenceEntity.getId());
|
||||
logger.debug("License '{}' migrated", licensePid);
|
||||
}
|
||||
|
||||
page++;
|
||||
DmpReferenceEntity dmpReferenceEntity = new DmpReferenceEntity();
|
||||
dmpReferenceEntity.setId(UUID.randomUUID());
|
||||
dmpReferenceEntity.setDmpId(item.getId());
|
||||
dmpReferenceEntity.setReferenceId(licenseIdByName.get(licensePid));
|
||||
dmpReferenceEntity.setCreatedAt(Instant.now());
|
||||
dmpReferenceEntity.setUpdatedAt(Instant.now());
|
||||
dmpReferenceEntity.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
|
||||
dmpReferenceEntity.setIsActive(IsActive.Active);
|
||||
|
||||
if (!licenseExists) entityManager.persist(referenceEntity);
|
||||
entityManager.persist(dmpReferenceEntity);
|
||||
}
|
||||
} while (items != null && !items.isEmpty() && !TestMode);
|
||||
logger.info("Dmp licenses migration finished");
|
||||
}
|
||||
this.entityManager.flush();
|
||||
|
||||
return licenseIdByName;
|
||||
}
|
||||
|
||||
private DmpReferenceDataEntity buildDmpReferenceDataEntity(SystemFieldEntity systemField){
|
||||
DmpReferenceDataEntity data = new DmpReferenceDataEntity();
|
||||
data.setBlueprintFieldId(systemField.getId());
|
||||
return data;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -21,24 +21,15 @@ public class ReferenceMigrationService {
|
|||
|
||||
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class));
|
||||
private final DatabaseRepository databaseRepository;
|
||||
private final QueryFactory queryFactory;
|
||||
private static final int PageSize = 500;
|
||||
private static final boolean TestMode = false;
|
||||
private final EntityManager entityManager;
|
||||
|
||||
public ReferenceMigrationService(DatabaseRepository databaseRepository, QueryFactory queryFactory, EntityManager entityManager) {
|
||||
public ReferenceMigrationService(DatabaseRepository databaseRepository, EntityManager entityManager) {
|
||||
this.databaseRepository = databaseRepository;
|
||||
this.queryFactory = queryFactory;
|
||||
this.entityManager = entityManager;
|
||||
}
|
||||
|
||||
public void migrateDmpReferences() {
|
||||
migrateDmpOrganizations();
|
||||
migrateDmpResearchers();
|
||||
migrateDmpProjects();
|
||||
migrateDmpGrantsAndFounders();
|
||||
}
|
||||
|
||||
public void migrateDatasetReferences() {
|
||||
migrateDatasetDataRepositories();
|
||||
migrateDatasetExternalDatasets();
|
||||
|
@ -46,149 +37,6 @@ public class ReferenceMigrationService {
|
|||
migrateDatasetServices();
|
||||
}
|
||||
|
||||
public void migrateDmpOrganizations() {
|
||||
OrganisationDao organisationDao = databaseRepository.getOrganisationDao();
|
||||
long total = organisationDao.asQueryable().count();
|
||||
logger.debug("Migrate Dmp Organisation (from Organization) Total : " + total);
|
||||
int page = 0;
|
||||
|
||||
List<Organisation> items;
|
||||
do {
|
||||
items = organisationDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
|
||||
if (items != null && !items.isEmpty()) {
|
||||
logger.debug("Migrate Dmp Organisation " + page * PageSize + " of " + total);
|
||||
|
||||
for (Organisation item : items) {
|
||||
// entityManager.detach(item);
|
||||
|
||||
for (DMP dmp : item.getDmps()) {
|
||||
DmpReferenceEntity data = new DmpReferenceEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
data.setDmpId(dmp.getId());
|
||||
data.setReferenceId(item.getId());
|
||||
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
|
||||
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
|
||||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
}
|
||||
}
|
||||
this.entityManager.flush();
|
||||
|
||||
page++;
|
||||
}
|
||||
} while (items != null && !items.isEmpty());
|
||||
}
|
||||
|
||||
public void migrateDmpResearchers() {
|
||||
ResearcherDao researcherDao = databaseRepository.getResearcherDao();
|
||||
long total = researcherDao.asQueryable().count();
|
||||
logger.debug("Migrate Dmp Researcher (from Researcher) Total : " + total);
|
||||
int page = 0;
|
||||
|
||||
List<Researcher> items;
|
||||
do {
|
||||
items = researcherDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
|
||||
if (items != null && !items.isEmpty()) {
|
||||
logger.debug("Migrate Dmp Researcher " + page * PageSize + " of " + total);
|
||||
|
||||
for (Researcher item : items) {
|
||||
// entityManager.detach(item);
|
||||
|
||||
for (DMP dmp : item.getdMPs()) {
|
||||
DmpReferenceEntity data = new DmpReferenceEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
data.setDmpId(dmp.getId());
|
||||
data.setReferenceId(item.getId());
|
||||
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
|
||||
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
|
||||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
}
|
||||
}
|
||||
this.entityManager.flush();
|
||||
|
||||
page++;
|
||||
}
|
||||
} while (items != null && !items.isEmpty());
|
||||
}
|
||||
|
||||
public void migrateDmpProjects() {
|
||||
ProjectDao projectDao = databaseRepository.getProjectDao();
|
||||
long total = projectDao.asQueryable().count();
|
||||
logger.debug("Migrate Dmp Project (from Project) Total : " + total);
|
||||
int page = 0;
|
||||
|
||||
List<Project> items;
|
||||
do {
|
||||
items = projectDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
|
||||
if (items != null && !items.isEmpty()) {
|
||||
logger.debug("Migrate Dmp Project " + page * PageSize + " of " + total);
|
||||
|
||||
for (Project item : items) {
|
||||
// entityManager.detach(item);
|
||||
|
||||
for (DMP dmp : item.getDmps()) {
|
||||
DmpReferenceEntity data = new DmpReferenceEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
data.setDmpId(dmp.getId());
|
||||
data.setReferenceId(item.getId());
|
||||
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
|
||||
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
|
||||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
}
|
||||
}
|
||||
this.entityManager.flush();
|
||||
|
||||
page++;
|
||||
}
|
||||
} while (items != null && !items.isEmpty());
|
||||
}
|
||||
|
||||
public void migrateDmpGrantsAndFounders() {
|
||||
GrantDao grantDao = databaseRepository.getGrantDao();
|
||||
long total = grantDao.asQueryable().count();
|
||||
logger.debug("Migrate Dmp Grants (from Grants) Total : " + total);
|
||||
int page = 0;
|
||||
|
||||
List<Grant> items;
|
||||
do {
|
||||
items = grantDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
|
||||
if (items != null && !items.isEmpty()) {
|
||||
logger.debug("Migrate Dmp Grants " + page * PageSize + " of " + total);
|
||||
|
||||
for (Grant item : items) {
|
||||
// entityManager.detach(item);
|
||||
|
||||
for (DMP dmp : item.getDmps()) {
|
||||
DmpReferenceEntity data = new DmpReferenceEntity();
|
||||
data.setId(UUID.randomUUID());
|
||||
data.setDmpId(dmp.getId());
|
||||
data.setReferenceId(item.getId());
|
||||
data.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
|
||||
data.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
|
||||
data.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(data);
|
||||
|
||||
if (item.getFunder() != null){
|
||||
DmpReferenceEntity founder = new DmpReferenceEntity();
|
||||
founder.setId(UUID.randomUUID());
|
||||
founder.setDmpId(dmp.getId());
|
||||
founder.setReferenceId(item.getFunder().getId());
|
||||
founder.setCreatedAt(dmp.getCreated() != null ? dmp.getCreated().toInstant() : Instant.now());
|
||||
founder.setUpdatedAt(dmp.getModified() != null ? dmp.getModified().toInstant() : Instant.now());
|
||||
founder.setIsActive(IsActive.Active);
|
||||
this.entityManager.persist(founder);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.entityManager.flush();
|
||||
|
||||
page++;
|
||||
}
|
||||
} while (items != null && !items.isEmpty());
|
||||
}
|
||||
|
||||
public void migrateDatasetDataRepositories() {
|
||||
DatasetDataRepositoryDao datasetDataRepositoryDao = databaseRepository.getDatasetDataRepositoryDao();
|
||||
long total = datasetDataRepositoryDao.asQueryable().count();
|
||||
|
|
|
@ -87,7 +87,6 @@ public class MigrationController {
|
|||
this.serviceMigrationService.migrate();
|
||||
|
||||
this.dmpMigrationService.migrate();
|
||||
this.dmpMigrationService.migrateDmpLicenses();
|
||||
this.dmpDatasetProfileMigrationService.migrate();
|
||||
this.datasetMigrationService.migrate();
|
||||
this.tagMigrationService.migrate();
|
||||
|
@ -95,7 +94,6 @@ public class MigrationController {
|
|||
this.dmpUserMigrationService.migrate();
|
||||
|
||||
this.referenceMigrationService.migrateDatasetReferences();
|
||||
this.referenceMigrationService.migrateDmpReferences();
|
||||
|
||||
this.userContactInfoMigrationService.migrate();
|
||||
this.userMigrationService.migrate();
|
||||
|
@ -121,18 +119,11 @@ public class MigrationController {
|
|||
|
||||
@GetMapping("dmps")
|
||||
@Transactional
|
||||
public boolean migrateDmps() throws JsonProcessingException, NoSuchFieldException, IllegalAccessException {
|
||||
public boolean migrateDmps() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException {
|
||||
this.dmpMigrationService.migrate();
|
||||
return true;
|
||||
}
|
||||
|
||||
@GetMapping("dmps/licenses")
|
||||
@Transactional
|
||||
public boolean migrateDmpLicenses() throws JsonProcessingException, NoSuchFieldException, IllegalAccessException {
|
||||
this.dmpMigrationService.migrateDmpLicenses();
|
||||
return true;
|
||||
}
|
||||
|
||||
@GetMapping("datasets")
|
||||
@Transactional
|
||||
public boolean migrateDatasets() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
|
||||
|
@ -169,13 +160,6 @@ public class MigrationController {
|
|||
return true;
|
||||
}
|
||||
|
||||
@GetMapping("dmp-references")
|
||||
@Transactional
|
||||
public boolean migrateDmpReferences() {
|
||||
this.referenceMigrationService.migrateDmpReferences();
|
||||
return true;
|
||||
}
|
||||
|
||||
@GetMapping("dmp-users")
|
||||
@Transactional
|
||||
public boolean migrateDmpUsers() {
|
||||
|
|
Loading…
Reference in New Issue