MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

migration changes

This commit is contained in:
Efstratios Giannopoulos 2024-02-21 18:38:28 +02:00
parent bc5f394ece
commit f7babf1f0b
25 changed files with 664 additions and 230 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dmp-backend/core/src/main/java/eu/eudat/commons/enums/FieldType.java
View File

@ -9,8 +9,8 @@ public enum FieldType implements DatabaseEnum<String> {
SELECT(Names.Select),
BOOLEAN_DECISION(Names.BooleanDecision),
RADIO_BOX(Names.RadioBox),
INTERNAL_ENTRIES_DMPS(Names.InternalDmpDmps),
INTERNAL_ENTRIES_DESCRIPTIONS(Names.InternalDmpDescriptions),
INTERNAL_ENTRIES_DMPS(Names.InternalEntitiesDmps),
INTERNAL_ENTRIES_DESCRIPTIONS(Names.InternalEntitiesDescriptions),
CHECK_BOX(Names.CheckBox),
FREE_TEXT(Names.FreeText),
TEXT_AREA(Names.TextArea),
@ -29,8 +29,8 @@ public enum FieldType implements DatabaseEnum<String> {
public static final String Select = "select";
public static final String BooleanDecision = "booleanDecision";
public static final String RadioBox = "radiobox";
public static final String InternalDmpDmps = "internalDmpDmps";
public static final String InternalDmpDescriptions = "internalDmpDescriptions";
public static final String InternalEntitiesDmps = "internalEntitiesDmps";
public static final String InternalEntitiesDescriptions = "internalEntitiesDescriptions";
public static final String CheckBox = "checkBox";
public static final String FreeText = "freetext";
public static final String TextArea = "textarea";

3
dmp-backend/core/src/main/java/eu/eudat/commons/types/descriptiontemplate/DefinitionEntity.java
View File

@ -43,9 +43,8 @@ public class DefinitionEntity {
public List<FieldSetEntity> getFieldSetById(String id) {
return this.getAllFieldSets().stream().filter(x-> id.equals(x.getId())).toList();
}
public List<FieldEntity> getFieldById(String id) {
return this.getAllField().stream().filter(x-> id.equals(x.getId())).toList();
}
}

4
dmp-backend/core/src/main/java/eu/eudat/model/persist/descriptiontemplatedefinition/fielddata/BaseFieldDataPersist.java
View File

@ -22,8 +22,8 @@ import java.util.List;
defaultImpl = LabelDataPersist.class)
@JsonSubTypes({
@JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.BooleanDecision),
@JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.InternalDmpDescriptions),
@JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.InternalDmpDmps),
@JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.InternalEntitiesDescriptions),
@JsonSubTypes.Type(value = LabelAndMultiplicityDataPersist.class, name = FieldType.Names.InternalEntitiesDmps),
@JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.CheckBox),
@JsonSubTypes.Type(value = LabelDataPersist.class, name = FieldType.Names.DatePicker),
@JsonSubTypes.Type(value = ExternalDatasetDataPersist.class, name = FieldType.Names.ExternalDatasets),

2
dmp-backend/core/src/main/java/eu/eudat/service/referencetype/ReferenceTypeServiceImpl.java
View File

@ -82,7 +82,7 @@ public class ReferenceTypeServiceImpl implements ReferenceTypeService {
data = this.entityManager.find(ReferenceTypeEntity.class, model.getId());
if (data == null)
throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), ReferenceType.class.getSimpleName()}, LocaleContextHolder.getLocale()));
if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(model.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(model.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
} else {
data = new ReferenceTypeEntity();

2
dmp-backend/core/src/main/java/eu/eudat/service/remotefetcher/RemoteFetcherServiceImpl.java
View File

@ -213,7 +213,7 @@ public class RemoteFetcherServiceImpl implements RemoteFetcherService {
if(externalDataResult != null) {
if (apiSource.getFilterType() != null && apiSource.getFilterType().equals("local") && (externalReferenceCriteria.getLike() != null && !externalReferenceCriteria.getLike().isEmpty())) {
externalDataResult.setResults(externalDataResult.getResults().stream()
.filter(r -> r.get("name").toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
.filter(r -> r.get(ReferenceEntity.KnownFields.Label).toLowerCase().contains(externalReferenceCriteria.getLike().toLowerCase()))
.collect(Collectors.toList()));
}
externalDataResult.setResults(externalDataResult.getResults().stream().peek(x -> x.put(ReferenceEntity.KnownFields.SourceLabel, apiSource.getLabel())).peek(x -> x.put(ReferenceEntity.KnownFields.Key, apiSource.getKey())).toList());

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/configurations/DevelDatabaseConfiguration.java
View File

@ -83,6 +83,8 @@ public class DevelDatabaseConfiguration {
properties.setProperty("hibernate.dialect", "org.hibernate.dialect.PostgreSQLDialect");
properties.setProperty("hibernate.show_sql", "false");
properties.setProperty("hibernate.temp.use_jdbc_metadata_defaults", "false");
properties.setProperty("hibernate.flushMode", "MANUAL");
properties.setProperty("org.hibernate.flushMode", "MANUAL");
return properties;
}
}

2
dmp-migration-tool/web/src/main/java/eu/old/eudat/configurations/ProductionDatabaseConfiguration.java
View File

@ -71,6 +71,8 @@ public class ProductionDatabaseConfiguration {
properties.setProperty("hibernate.dialect", "org.hibernate.dialect.PostgreSQL92Dialect");
properties.setProperty("hibernate.show_sql", "true");
properties.setProperty("hibernate.temp.use_jdbc_metadata_defaults", "false");
properties.setProperty("hibernate.flushMode", "MANUAL");
properties.setProperty("org.hibernate.flushMode", "MANUAL");
return properties;
}
}

17
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DataRepositoryMigrationService.java
View File

@ -4,7 +4,6 @@ import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
@ -17,8 +16,10 @@ import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.UUID;
@Service
public class DataRepositoryMigrationService {
@ -58,29 +59,31 @@ public class DataRepositoryMigrationService {
String[] referenceParts = item.getReference().split(":", 2);
boolean isInternal = referenceParts[0].equals(InternalReferenceSource);
ReferenceEntity data = new ReferenceEntity();
data.setId(item.getId());
data.setLabel(item.getLabel());
data.setAbbreviation(item.getAbbreviation());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.DataRepositories);
data.setTypeId(ReferenceTypeIds.DataRepositories);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId());
data.setReference(referenceParts[1]);
data.setSource(isInternal? "Internal" : referenceParts[0]);
data.setSourceType(isInternal? ReferenceSourceType.Internal : ReferenceSourceType.External);
DefinitionEntity definitionEntity = new DefinitionEntity();
definitionEntity.setFields(new ArrayList<>());
if (!this.conventionService.isNullOrEmpty(item.getUri())){
DefinitionEntity definitionEntity = new DefinitionEntity();
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setCode("uri");
fieldEntity.setDataType(ReferenceFieldDataType.Text);
fieldEntity.setValue(item.getUri());
definitionEntity.setFields(List.of(fieldEntity));
data.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
definitionEntity.getFields().add(fieldEntity);
}
if (!definitionEntity.getFields().isEmpty()) data.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
this.entityManager.persist(data);
}

116
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java
View File

@ -6,16 +6,16 @@ import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.*;
import eu.eudat.commons.types.description.FieldEntity;
import eu.eudat.commons.types.description.PropertyDefinitionEntity;
import eu.eudat.commons.types.description.PropertyDefinitionFieldSetEntity;
import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
import eu.eudat.commons.types.dmp.DmpContactEntity;
import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
import eu.eudat.commons.types.dmpblueprint.DefinitionEntity;
import eu.eudat.data.DescriptionEntity;
import eu.eudat.data.DmpBlueprintEntity;
import eu.eudat.data.DmpDescriptionTemplateEntity;
import eu.eudat.data.DmpEntity;
import eu.eudat.data.*;
import eu.eudat.model.Dmp;
import eu.eudat.model.DmpDescriptionTemplate;
import eu.eudat.query.DescriptionTemplateQuery;
import eu.eudat.query.DmpBlueprintQuery;
import eu.eudat.query.DmpDescriptionTemplateQuery;
import eu.eudat.query.DmpQuery;
@ -83,9 +83,14 @@ public class DatasetMigrationService {
dmpBlueprintsMap.put(dmpBlueprint.getId(), definitionEntity);
}
List<DmpDescriptionTemplateEntity> descriptionTemplates = this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collect();
List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities = this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collect();
List<DescriptionTemplateEntity> descriptionTemplates = this.queryFactory.query(DescriptionTemplateQuery.class).ids(items.stream().map(x-> x.getProfile().getId()).distinct().toList()).collect();
Map<UUID, UUID> creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId));;
Map<UUID, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity> descriptionTemplateDefinitionMap = new HashMap<>();
for (DescriptionTemplateEntity descriptionTemplateEntity : descriptionTemplates){
descriptionTemplateDefinitionMap.put(descriptionTemplateEntity.getId(), this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition()));
}
for (Dataset item : items) {
//entityManager.detach(item);
@ -104,24 +109,7 @@ public class DatasetMigrationService {
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = descriptionTemplates.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
if (descriptionTemplates.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) {
DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity();
dmpDescriptionTemplateEntity.setId(UUID.randomUUID());
dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId());
dmpDescriptionTemplateEntity.setDmpId(item.getDmp().getId());
dmpDescriptionTemplateEntity.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setSectionId(sectionId);
dmpDescriptionTemplateEntity.setIsActive(IsActive.Active);
this.entityManager.persist(dmpDescriptionTemplateEntity);
itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity);
} else {
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
}
}
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = this.getOrCreateDmpDescriptionTemplateEntity(item, sectionId, dmpDescriptionTemplateEntities);
if (itemDescriptionTemplates.size() > 1) {
logger.error("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
@ -138,7 +126,7 @@ public class DatasetMigrationService {
}
data.setDmpId(item.getDmp().getId());
data.setLabel(item.getLabel());
data.setDmpDescriptionTemplateId(itemDescriptionTemplates.get(0).getId());
data.setDmpDescriptionTemplateId(itemDescriptionTemplates.getFirst().getId());
data.setDescriptionTemplateId(item.getProfile().getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
@ -152,22 +140,9 @@ public class DatasetMigrationService {
data.setStatus(DescriptionStatus.of(item.getStatus()));
}
if (item.getProperties() != null) {
JSONObject jObject = new JSONObject(item.getProperties());
Map<String, Object> properties = jObject.toMap();
PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity();
List<FieldEntity> fieldEntities = new ArrayList<>();
for (String key: properties.keySet() ) {
FieldEntity fieldEntity = new FieldEntity();
fieldEntity.setKey(key);
fieldEntity.setValue(properties.get(key) != null ? properties.get(key).toString(): null);
fieldEntities.add(fieldEntity);
}
propertyDefinitionEntity.setFields(fieldEntities);
data.setProperties(this.jsonHandlingService.toJson(propertyDefinitionEntity));
}
eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null);
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity)));
if (data.getCreatedById() == null){
logger.error("Migration skipped creator not found " + item.getId());
@ -182,5 +157,66 @@ public class DatasetMigrationService {
}
} while (items != null && !items.isEmpty() && !TestMode);
}
private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
if (dmpDescriptionTemplateEntities.stream().filter(x-> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).count() > 0) {
DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity();
dmpDescriptionTemplateEntity.setId(UUID.randomUUID());
dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId());
dmpDescriptionTemplateEntity.setDmpId(item.getDmp().getId());
dmpDescriptionTemplateEntity.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setSectionId(sectionId);
dmpDescriptionTemplateEntity.setIsActive(IsActive.Active);
this.entityManager.persist(dmpDescriptionTemplateEntity);
itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity);
} else {
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
}
}
return itemDescriptionTemplates;
}
private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity) {
if (item.getProperties() != null) return null;
JSONObject jObject = new JSONObject(item.getProperties());
Map<String, Object> properties = jObject.toMap();
PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity();
List<FieldSetEntity> fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets();
// for (String key : properties.keySet()) {
// FieldSetEntity currentFieldSet = null;
// eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null;
// for (FieldSetEntity fieldSetEntity : fieldSetEntities){
// List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(key);
// if (!fieldEntities.isEmpty()){
// currentFieldSet = fieldSetEntity;
// currentField = fieldEntities.getFirst();
// }
// }
//
// if (currentFieldSet == null || currentField == null) throw new MyApplicationException("aaa");
//
// if (!propertyDefinitionEntity.getFieldSets().containsKey(currentFieldSet.getId())){
// PropertyDefinitionFieldSetEntity fieldSetEntity = new PropertyDefinitionFieldSetEntity();
// fieldSetEntity.setItems(new ArrayList<>());
// propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), fieldSetEntity);
// }
// propertyDefinitionEntity.getFieldSets().get(currentFieldSet.getId())
//
// FieldEntity fieldEntity = new FieldEntity();
// fieldEntity.setKey(key);
// fieldEntity.setValue(properties.get(key) != null ? properties.get(key).toString() : null);
// fieldEntities.add(fieldEntity);
// propertyDefinitionEntity.getFieldSets().put(key, fieldEntity);
// }
// propertyDefinitionEntity.setFields(fieldEntities);
return propertyDefinitionEntity;
}
}

207
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlCleanInvalidReferenceTypesService.java Normal file
View File

@ -0,0 +1,207 @@
package eu.old.eudat.migration;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.FieldType;
import eu.eudat.commons.types.descriptiontemplate.*;
import eu.eudat.commons.types.descriptiontemplate.fielddata.BaseFieldDataEntity;
import eu.eudat.commons.types.descriptiontemplate.fielddata.LabelDataEntity;
import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.DescriptionTemplateEntity;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.DescriptionTemplate;
import eu.eudat.model.ReferenceType;
import eu.eudat.query.DescriptionTemplateQuery;
import eu.eudat.query.ReferenceTypeQuery;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.Paging;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.logging.LoggerService;
import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;
import javax.management.InvalidApplicationException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@Service
public class DescriptionTemplateXmlCleanInvalidReferenceTypesService {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DescriptionTemplateXmlCleanInvalidReferenceTypesService.class));
private final EntityManager entityManager;
private final ConventionService conventionService;
private final XmlHandlingService xmlHandlingService;
private final QueryFactory queryFactory;
private static final int PageSize = 500;
private static final boolean TestMode = false;
public DescriptionTemplateXmlCleanInvalidReferenceTypesService(EntityManager entityManager, ConventionService conventionService, XmlHandlingService xmlHandlingService, QueryFactory queryFactory) {
this.entityManager = entityManager;
this.conventionService = conventionService;
this.xmlHandlingService = xmlHandlingService;
this.queryFactory = queryFactory;
}
public void migrate() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException, InvalidApplicationException, TransformerException, URISyntaxException {
long total = this.queryFactory.query(DescriptionTemplateQuery.class).count();
logger.debug("Clean old reference types from DescriptionTemplate Total : " + total);
int page = 0;
List<UUID> existingReferenceTypeIds = this.queryFactory.query(ReferenceTypeQuery.class).collectAs(new BaseFieldSet().ensure(ReferenceType._id)).stream().map(ReferenceTypeEntity::getId).toList();
List<DescriptionTemplateEntity> items;
do {
DescriptionTemplateQuery query = this.queryFactory.query(DescriptionTemplateQuery.class);
query.setOrder(new Ordering().addDescending(DescriptionTemplate._createdAt));
query.setPage(new Paging(page * PageSize, PageSize));
items = query.collect();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate DescriptionTemplate " + page * PageSize + " of " + total);
for (DescriptionTemplateEntity item : items) {
if (this.conventionService.isNullOrEmpty(item.getDefinition())) continue;
DefinitionEntity definition = this.xmlHandlingService.fromXml(DefinitionEntity.class, item.getDefinition());
item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(definition, existingReferenceTypeIds)));
this.entityManager.merge(item);
}
this.entityManager.flush();
page++;
}
} while (items != null && !items.isEmpty() && !TestMode);
this.entityManager.flush();
}
private @NotNull DefinitionEntity buildDefinitionEntity(DefinitionEntity persist, List<UUID> existingReferenceTypeIds) {
DefinitionEntity data = new DefinitionEntity();
if (persist == null)
return data;
if (!this.conventionService.isListNullOrEmpty(persist.getPages())) {
data.setPages(new ArrayList<>());
for (PageEntity pagePersist : persist.getPages()) {
data.getPages().add(this.buildPageEntity(pagePersist, existingReferenceTypeIds));
}
}
return data;
}
private @NotNull SectionEntity buildSectionEntity(SectionEntity persist, List<UUID> existingReferenceTypeIds) {
SectionEntity data = new SectionEntity();
if (persist == null)
return data;
data.setId(persist.getId());
data.setDescription(persist.getDescription());
data.setExtendedDescription(persist.getExtendedDescription());
data.setNumbering(persist.getNumbering());
data.setOrdinal(persist.getOrdinal());
data.setDefaultVisibility(persist.isDefaultVisibility());
data.setMultiplicity(persist.getMultiplicity());
data.setTitle(persist.getTitle());
if (!this.conventionService.isListNullOrEmpty(persist.getSections())) {
data.setSections(new ArrayList<>());
for (SectionEntity sectionPersist : persist.getSections()) {
data.getSections().add(this.buildSectionEntity(sectionPersist, existingReferenceTypeIds));
}
}
if (!this.conventionService.isListNullOrEmpty(persist.getFieldSets())) {
data.setFieldSets(new ArrayList<>());
for (FieldSetEntity fieldSetPersist : persist.getFieldSets()) {
data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist, existingReferenceTypeIds));
}
}
return data;
}
private @NotNull FieldSetEntity buildFieldSetEntity(FieldSetEntity persist, List<UUID> existingReferenceTypeIds) {
FieldSetEntity data = new FieldSetEntity();
if (persist == null)
return data;
data.setId(persist.getId());
data.setOrdinal(persist.getOrdinal());
data.setNumbering(persist.getNumbering());
data.setTitle(persist.getTitle());
data.setExtendedDescription(persist.getExtendedDescription());
data.setAdditionalInformation(persist.getAdditionalInformation());
data.setHasCommentField(persist.getHasCommentField());
data.setMultiplicity(persist.getMultiplicity());
if (!this.conventionService.isListNullOrEmpty(persist.getFields())) {
data.setFields(new ArrayList<>());
for (FieldEntity fieldPersist : persist.getFields()) {
data.getFields().add(this.buildFieldEntity(fieldPersist, existingReferenceTypeIds));
}
}
return data;
}
private @NotNull FieldEntity buildFieldEntity(FieldEntity persist, List<UUID> existingReferenceTypeIds) {
FieldEntity data = new FieldEntity();
if (persist == null)
return data;
data.setId(persist.getId());
data.setOrdinal(persist.getOrdinal());
data.setSchematics(persist.getSchematics());
data.setNumbering(persist.getNumbering());
data.setDefaultValue(persist.getDefaultValue());
data.setValidations(persist.getValidations());
data.setIncludeInExport(persist.getIncludeInExport());
data.setVisibilityRules(persist.getVisibilityRules());
if (persist.getData() != null) data.setData(this.buildFieldDataEntity(persist.getData(), existingReferenceTypeIds));
return data;
}
private BaseFieldDataEntity buildFieldDataEntity(BaseFieldDataEntity persist, List<UUID> existingReferenceTypeIds) {
if (persist.getFieldType().equals(FieldType.REFERENCE_TYPES)){
ReferenceTypeDataEntity entity = (ReferenceTypeDataEntity)persist;
if (existingReferenceTypeIds.contains(entity.getReferenceTypeId())) return entity;
else {
LabelDataEntity data = new LabelDataEntity();
data.setLabel(entity.getLabel());
data.setFieldType(FieldType.TEXT_AREA);
return data;
}
} else {
return persist;
}
}
private @NotNull PageEntity buildPageEntity(PageEntity persist, List<UUID> existingReferenceTypeIds) {
PageEntity data = new PageEntity();
if (persist == null)
return data;
data.setId(persist.getId());
data.setOrdinal(persist.getOrdinal());
data.setTitle(persist.getTitle());
if (!this.conventionService.isListNullOrEmpty(persist.getSections())) {
data.setSections(new ArrayList<>());
for (SectionEntity sectionPersist : persist.getSections()) {
data.getSections().add(this.buildSectionEntity(sectionPersist, existingReferenceTypeIds));
}
}
return data;
}
}

296
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DescriptionTemplateXmlMigrationService.java
View File

@ -1,13 +1,14 @@
package eu.old.eudat.migration;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.FieldDataExternalDatasetType;
import eu.eudat.commons.enums.FieldType;
import eu.eudat.commons.enums.FieldValidationType;
import eu.eudat.commons.enums.*;
import eu.eudat.commons.types.descriptiontemplate.*;
import eu.eudat.commons.types.descriptiontemplate.fielddata.*;
import eu.eudat.commons.types.referencetype.*;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.DescriptionTemplateEntity;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.ReferenceTypeEntity;
import eu.eudat.model.DescriptionTemplate;
import eu.eudat.query.DescriptionTemplateQuery;
import eu.old.eudat.logic.utilities.builders.XmlBuilder;
@ -24,6 +25,7 @@ import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Service;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;
@ -32,10 +34,11 @@ import javax.management.InvalidApplicationException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
import java.util.UUID;
import java.net.URI;
import java.net.URISyntaxException;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;
@Service
public class DescriptionTemplateXmlMigrationService {
@ -56,18 +59,18 @@ public class DescriptionTemplateXmlMigrationService {
this.queryFactory = queryFactory;
}
public void migrate() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException, InvalidApplicationException, TransformerException {
public void migrate() throws IOException, NoSuchFieldException, IllegalAccessException, JAXBException, ParserConfigurationException, InstantiationException, SAXException, InvalidApplicationException, TransformerException, URISyntaxException {
long total = this.queryFactory.query(DescriptionTemplateQuery.class).count();
logger.debug("Migrate DescriptionTemplate Total : " + total);
int page = 0;
Map<String, ReferenceTypeEntity> referenceTypeEntityMap = new HashMap<>();
List<DescriptionTemplateEntity> items;
do {
DescriptionTemplateQuery query = this.queryFactory.query(DescriptionTemplateQuery.class);
query.setOrder(new Ordering().addDescending(DescriptionTemplate._createdAt));
query.setPage(new Paging(page * PageSize, PageSize));
items = query.collect();
if (items != null && !items.isEmpty()) {
logger.debug("Migrate DescriptionTemplate " + page * PageSize + " of " + total);
@ -86,7 +89,7 @@ public class DescriptionTemplateXmlMigrationService {
logger.error("Migrate DescriptionTemplate " + item.getId() + " failed read xml");
continue;
}
item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel)));
item.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(viewStyleModel, referenceTypeEntityMap)));
this.entityManager.merge(item);
}
@ -94,23 +97,29 @@ public class DescriptionTemplateXmlMigrationService {
page++;
}
} while (items != null && !items.isEmpty() && !TestMode);
logger.debug("Migrate DescriptionTemplate ReferenceTypes : " + referenceTypeEntityMap.size());
for (ReferenceTypeEntity item : referenceTypeEntityMap.values()){
this.entityManager.persist(item);
}
this.entityManager.flush();
}
private @NotNull DefinitionEntity buildDefinitionEntity(ViewStyleModel persist) {
private @NotNull DefinitionEntity buildDefinitionEntity(ViewStyleModel persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
DefinitionEntity data = new DefinitionEntity();
if (persist == null)
return data;
if (!this.conventionService.isListNullOrEmpty(persist.getPages())) {
data.setPages(new ArrayList<>());
for (Page pagePersist : persist.getPages()) {
data.getPages().add(this.buildPageEntity(pagePersist, persist.getSections()));
data.getPages().add(this.buildPageEntity(pagePersist, persist.getSections(), referenceTypeEntityMap));
}
}
return data;
}
private @NotNull SectionEntity buildSectionEntity(Section persist) {
private @NotNull SectionEntity buildSectionEntity(Section persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
SectionEntity data = new SectionEntity();
if (persist == null)
return data;
@ -127,21 +136,21 @@ public class DescriptionTemplateXmlMigrationService {
if (!this.conventionService.isListNullOrEmpty(persist.getSections())) {
data.setSections(new ArrayList<>());
for (Section sectionPersist : persist.getSections()) {
data.getSections().add(this.buildSectionEntity(sectionPersist));
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap));
}
}
if (!this.conventionService.isListNullOrEmpty(persist.getFieldSets())) {
data.setFieldSets(new ArrayList<>());
for (FieldSet fieldSetPersist : persist.getFieldSets()) {
data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist));
data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist, referenceTypeEntityMap));
}
}
return data;
}
private @NotNull FieldSetEntity buildFieldSetEntity(FieldSet persist) {
private @NotNull FieldSetEntity buildFieldSetEntity(FieldSet persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
FieldSetEntity data = new FieldSetEntity();
if (persist == null)
return data;
@ -158,13 +167,13 @@ public class DescriptionTemplateXmlMigrationService {
if (!this.conventionService.isListNullOrEmpty(persist.getFields())) {
data.setFields(new ArrayList<>());
for (Field fieldPersist : persist.getFields()) {
data.getFields().add(this.buildFieldEntity(fieldPersist));
data.getFields().add(this.buildFieldEntity(fieldPersist, referenceTypeEntityMap));
}
}
return data;
}
private @NotNull FieldEntity buildFieldEntity(Field persist) {
private @NotNull FieldEntity buildFieldEntity(Field persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
FieldEntity data = new FieldEntity();
if (persist == null)
return data;
@ -191,7 +200,7 @@ public class DescriptionTemplateXmlMigrationService {
}).toList());
data.setIncludeInExport(persist.getExport());
if (persist.getData() != null)
data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle()));
data.setData(this.buildFieldDataEntity(persist.getData(), persist.getViewStyle().getRenderStyle(), referenceTypeEntityMap));
if (persist.getVisible() != null && !this.conventionService.isListNullOrEmpty(persist.getVisible().getRules())) {
data.setVisibilityRules(new ArrayList<>());
@ -202,7 +211,7 @@ public class DescriptionTemplateXmlMigrationService {
return data;
}
private BaseFieldDataEntity buildFieldDataEntity(FieldData persist, String renderStyle) {
private BaseFieldDataEntity buildFieldDataEntity(FieldData<?> persist, String renderStyle, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
if (renderStyle.equals("checkBox")){
return this.buildLabelDataEntity(persist.getLabel(), FieldType.CHECK_BOX);
}
@ -231,34 +240,34 @@ public class DescriptionTemplateXmlMigrationService {
return this.buildLabelDataEntity(persist.getLabel(), FieldType.CURRENCY);
}
if (renderStyle.equals("taxonomies")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((TaxonomiesData)persist).getMultiAutoComplete(),FieldType.TAXONOMIES);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((TaxonomiesData)persist).getMultiAutoComplete(), ReferenceTypeIds.Taxonomy);
}
if (renderStyle.equals("services")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((ServicesData)persist).getMultiAutoComplete(),FieldType.SERVICES);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((ServicesData)persist).getMultiAutoComplete(),ReferenceTypeIds.Services);
}
if (renderStyle.equals("researchers")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((ResearcherData)persist).getMultiAutoComplete(),FieldType.RESEARCHERS);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((ResearcherData)persist).getMultiAutoComplete(),ReferenceTypeIds.Researcher);
}
if (renderStyle.equals("registries")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((RegistriesData)persist).getMultiAutoComplete(),FieldType.REGISTRIES);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((RegistriesData)persist).getMultiAutoComplete(),ReferenceTypeIds.Registries);
}
if (renderStyle.equals("pubRepositories")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((DataRepositoriesData)persist).getMultiAutoComplete(),FieldType.PUB_REPOSITORIES);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((DataRepositoriesData)persist).getMultiAutoComplete(),ReferenceTypeIds.PubRepositories);
}
if (renderStyle.equals("publications")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((PublicationsData)persist).getMultiAutoComplete(),FieldType.PUB_REPOSITORIES);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((PublicationsData)persist).getMultiAutoComplete(), ReferenceTypeIds.Publication);
}
if (renderStyle.equals("organizations")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((OrganizationsData)persist).getMultiAutoComplete(),FieldType.ORGANIZATIONS);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((OrganizationsData)persist).getMultiAutoComplete(), ReferenceTypeIds.Organizations);
}
if (renderStyle.equals("licenses")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((LicensesData)persist).getMultiAutoComplete(),FieldType.LICENSES);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((LicensesData)persist).getMultiAutoComplete(), ReferenceTypeIds.License);
}
if (renderStyle.equals("journalRepositories")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((DataRepositoriesData)persist).getMultiAutoComplete(),FieldType.JOURNAL_REPOSITORIES);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((DataRepositoriesData)persist).getMultiAutoComplete(), ReferenceTypeIds.Journal);
}
if (renderStyle.equals("dataRepositories")){
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((DataRepositoriesData)persist).getMultiAutoComplete(),FieldType.DATA_REPOSITORIES);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((DataRepositoriesData)persist).getMultiAutoComplete(), ReferenceTypeIds.DataRepositories);
}
if (renderStyle.equals("booleanDecision")){
return this.buildLabelDataEntity(persist.getLabel(), FieldType.BOOLEAN_DECISION);
@ -266,11 +275,11 @@ public class DescriptionTemplateXmlMigrationService {
if (renderStyle.equals("internalDmpEntities")){
switch (((InternalDmpEntitiesData<?>)persist).getType()){
case "dmps":
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((DMPsAutoCompleteData)persist).getMultiAutoComplete(),FieldType.INTERNAL_DMP_ENTRIES_DMPS);
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((DMPsAutoCompleteData)persist).getMultiAutoComplete(),FieldType.INTERNAL_ENTRIES_DMPS);
case "datasets":
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((DatasetsAutoCompleteData)persist).getMultiAutoComplete(),FieldType.INTERNAL_DMP_ENTRIES_DATASETS);
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((DatasetsAutoCompleteData)persist).getMultiAutoComplete(),FieldType.INTERNAL_ENTRIES_DESCRIPTIONS);
case "researchers":
return this.buildLabelAndMultiplicityDataEntity(persist.getLabel(), ((ResearchersAutoCompleteData)persist).getMultiAutoComplete(),FieldType.INTERNAL_DMP_ENTRIES_RESEARCHERS);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((ResearchersAutoCompleteData)persist).getMultiAutoComplete(), ReferenceTypeIds.Researcher);
}
}
@ -337,38 +346,8 @@ public class DescriptionTemplateXmlMigrationService {
return data;
}
case "autocomplete":{
ExternalSelectDataEntity data = new ExternalSelectDataEntity();
data.setLabel(persist.getLabel());
data.setSources(new ArrayList<>());
data.setMultipleSelect(((AutoCompleteData)persist).getMultiAutoComplete());
if (((AutoCompleteData)persist).getAutoCompleteSingleDataList() != null){
for (AutoCompleteData.AutoCompleteSingleData autoCompleteSingleData : ((AutoCompleteData)persist).getAutoCompleteSingleDataList()){
ExternalSelectDataEntity.ExternalSelectSourceEntity autoCompleteSingleDataEntity = new ExternalSelectDataEntity.ExternalSelectSourceEntity();
autoCompleteSingleDataEntity.setOptionsRoot(autoCompleteSingleData.getOptionsRoot());
autoCompleteSingleDataEntity.setUrl(autoCompleteSingleData.getUrl());
autoCompleteSingleDataEntity.setMethod(autoCompleteSingleData.getMethod());
autoCompleteSingleDataEntity.setHasAuth(autoCompleteSingleData.getHasAuth());
if (autoCompleteSingleData.getHasAuth() && autoCompleteSingleData.getAuth()!= null) {
ExternalSelectDataEntity.ExternalSelectAuthDataEntity optionEntity = new ExternalSelectDataEntity.ExternalSelectAuthDataEntity();
optionEntity.setBody(autoCompleteSingleData.getAuth().getBody());
optionEntity.setMethod(autoCompleteSingleData.getAuth().getMethod());
optionEntity.setUrl(autoCompleteSingleData.getAuth().getUrl());
optionEntity.setType(autoCompleteSingleData.getAuth().getType());
optionEntity.setBody(autoCompleteSingleData.getAuth().getBody());
autoCompleteSingleDataEntity.setAuth(optionEntity);
}
if (autoCompleteSingleData.getAutoCompleteOptions() != null) {
ExternalSelectDataEntity.ExternalSelectSourceBindingEntity optionEntity = new ExternalSelectDataEntity.ExternalSelectSourceBindingEntity();
optionEntity.setValue(autoCompleteSingleData.getAutoCompleteOptions().getValue());
optionEntity.setLabel(autoCompleteSingleData.getAutoCompleteOptions().getLabel());
optionEntity.setSource(autoCompleteSingleData.getAutoCompleteOptions().getSource());
autoCompleteSingleDataEntity.setSourceBinding(optionEntity);
}
data.getSources().add(autoCompleteSingleDataEntity);
}
}
data.setFieldType(FieldType.EXTERNAL_SELECT);
return data;
ReferenceTypeEntity referenceType = this.resolveReferenceTypeEntityAutoCompleteData(((AutoCompleteData)persist), referenceTypeEntityMap);
return this.buildReferenceTypeDataEntity(persist.getLabel(), ((AutoCompleteData)persist).getMultiAutoComplete(), referenceType.getId());
}
}
}
@ -390,6 +369,16 @@ public class DescriptionTemplateXmlMigrationService {
return data;
}
private BaseFieldDataEntity buildReferenceTypeDataEntity(String label, Boolean multiAutoComplete, UUID referenceTypeId) {
ReferenceTypeDataEntity data = new ReferenceTypeDataEntity();
data.setLabel(label);
data.setFieldType(FieldType.REFERENCE_TYPES);
data.setMultipleSelect(multiAutoComplete);
data.setReferenceTypeId(referenceTypeId);
return data;
}
private @NotNull RuleEntity buildRuleEntity(Rule persist) {
RuleEntity data = new RuleEntity();
if (persist == null)
@ -413,7 +402,7 @@ public class DescriptionTemplateXmlMigrationService {
private @NotNull PageEntity buildPageEntity(Page persist, List<Section> allSections) {
private @NotNull PageEntity buildPageEntity(Page persist, List<Section> allSections, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
PageEntity data = new PageEntity();
if (persist == null)
return data;
@ -426,7 +415,7 @@ public class DescriptionTemplateXmlMigrationService {
if (!this.conventionService.isListNullOrEmpty(sections)) {
data.setSections(new ArrayList<>());
for (Section sectionPersist : sections) {
data.getSections().add(this.buildSectionEntity(sectionPersist));
data.getSections().add(this.buildSectionEntity(sectionPersist, referenceTypeEntityMap));
}
}
return data;
@ -443,4 +432,177 @@ public class DescriptionTemplateXmlMigrationService {
}
}
}
public ReferenceTypeEntity resolveReferenceTypeEntityAutoCompleteData(AutoCompleteData persist, Map<String, ReferenceTypeEntity> referenceTypeEntityMap) throws URISyntaxException {
ReferenceTypeDefinitionEntity definitionEntity = this.buildReferenceTypeDefinitionEntity(persist);
String hash = this.referenceTypeCalculatedHash(definitionEntity);
ReferenceTypeEntity data = referenceTypeEntityMap.getOrDefault(hash, null);
if (data == null) {
data = new ReferenceTypeEntity();
data.setId(UUID.randomUUID());
data.setIsActive(IsActive.Active);
data.setCreatedAt(Instant.now());
data.setName(persist.getLabel().substring(0, Math.min(persist.getLabel().length(), 199)));
data.setCode(persist.getLabel().substring(0, Math.min(persist.getLabel().length(), 99)));
data.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
data.setUpdatedAt(Instant.now());
referenceTypeEntityMap.put(hash, data);
}
return data;
}
public String referenceTypeCalculatedHash(ReferenceTypeDefinitionEntity definitionEntity) {
List<String> sourceKeys = new ArrayList<>();
if (definitionEntity.getSources() != null){
for (ReferenceTypeSourceExternalApiConfigurationEntity source : definitionEntity.getSources().stream().map(x-> (ReferenceTypeSourceExternalApiConfigurationEntity)x).toList()){
StringBuilder sourceStringBuilder = new StringBuilder();
sourceStringBuilder.append(source.getUrl());
sourceStringBuilder.append(source.getHttpMethod().getValue().toString());
sourceStringBuilder.append(source.getKey());
if (source.getResults() != null){
sourceStringBuilder.append(source.getResults().getResultsArrayPath());
if (source.getResults().getFieldsMapping() != null) {
sourceStringBuilder.append(source.getResults().getFieldsMapping().stream().filter(x-> x.getCode().equals(ReferenceEntity.KnownFields.ReferenceId)).map(ResultFieldsMappingConfigurationEntity::getResponsePath).findFirst().orElse(""));
sourceStringBuilder.append(source.getResults().getFieldsMapping().stream().filter(x-> x.getCode().equals(ReferenceEntity.KnownFields.Label)).map(ResultFieldsMappingConfigurationEntity::getResponsePath).findFirst().orElse(""));
}
}
sourceKeys.add(sourceStringBuilder.toString().toLowerCase(Locale.ROOT));
}
}
return sourceKeys.stream().sorted().collect(Collectors.joining(""));
}
private @NotNull ReferenceTypeDefinitionEntity buildReferenceTypeDefinitionEntity(AutoCompleteData persist) throws URISyntaxException {
ReferenceTypeDefinitionEntity data = new ReferenceTypeDefinitionEntity();
if (persist == null) return data;
data.setFields(new ArrayList<>());
if (!this.conventionService.isListNullOrEmpty(persist.getAutoCompleteSingleDataList())){
data.setSources(new ArrayList<>());
int ordinal = 0;
for (AutoCompleteData.AutoCompleteSingleData sourceBaseConfigPersist: persist.getAutoCompleteSingleDataList()) {
data.getSources().add(this.buildSourceBaseConfigEntity(sourceBaseConfigPersist, ordinal));
ordinal++;
}
}
return data;
}
private @NotNull ReferenceTypeSourceBaseConfigurationEntity buildSourceBaseConfigEntity(AutoCompleteData.AutoCompleteSingleData persist, int ordinal) throws URISyntaxException {
if (persist == null) return new ReferenceTypeSourceExternalApiConfigurationEntity();
ReferenceTypeSourceExternalApiConfigurationEntity apiEntity = new ReferenceTypeSourceExternalApiConfigurationEntity();
URI uri;
if (persist.getUrl().contains("?")) {
uri = new URI(persist.getUrl().substring(0, persist.getUrl().trim().lastIndexOf("?")));
} else {
uri = new URI(persist.getUrl().trim());
}
String source = persist.getAutoCompleteOptions().getSource();
source = source != null && !source.isEmpty() ? source : uri.getHost();
String parsedUrl = persist.getUrl().trim();
parsedUrl = parsedUrl.replace("%20", " ");
parsedUrl = parsedUrl.replace("%22", "\"");
while (parsedUrl.contains("&amp;")) {
parsedUrl = parsedUrl.replace("&amp;", "&");
}
apiEntity.setUrl(parsedUrl);
apiEntity.setResults(this.buildResultsConfigEntity(persist));
apiEntity.setPaginationPath(null);
apiEntity.setContentType(MediaType.APPLICATION_JSON_VALUE);
apiEntity.setFirstPage("1");
apiEntity.setHttpMethod(this.toReferenceTypeExternalApiHTTPMethodType(persist.getMethod()));
apiEntity.setRequestBody(null);
apiEntity.setFilterType(null);
if (persist.getHasAuth() && persist.getAuth() != null) {
apiEntity.setAuth(this.buildAuthConfigEntity(persist.getAuth()));
}
apiEntity.setQueries(new ArrayList<>());
apiEntity.getQueries().add(this.buildQueryConfigEntity(parsedUrl));
apiEntity.setType(ReferenceTypeSourceType.API);
apiEntity.setKey(source);
apiEntity.setLabel(source);
apiEntity.setOrdinal(ordinal);
apiEntity.setReferenceTypeDependencyIds(null);
return apiEntity;
}
private @NotNull ResultsConfigurationEntity buildResultsConfigEntity(AutoCompleteData.AutoCompleteSingleData persist){
ResultsConfigurationEntity data = new ResultsConfigurationEntity();
data.setResultsArrayPath(persist.getOptionsRoot());
if (persist.getAutoCompleteOptions() == null && this.conventionService.isNullOrEmpty(persist.getAutoCompleteOptions().getLabel())) {
data.setFieldsMapping(new ArrayList<>());
ResultFieldsMappingConfigurationEntity labelField = new ResultFieldsMappingConfigurationEntity();
labelField.setCode(ReferenceEntity.KnownFields.Label);
labelField.setResponsePath(persist.getAutoCompleteOptions().getLabel());
data.getFieldsMapping().add(labelField);
}
if (persist.getAutoCompleteOptions() == null && this.conventionService.isNullOrEmpty(persist.getAutoCompleteOptions().getValue())) {
ResultFieldsMappingConfigurationEntity idField = new ResultFieldsMappingConfigurationEntity();
idField.setCode(ReferenceEntity.KnownFields.ReferenceId);
idField.setResponsePath(persist.getAutoCompleteOptions().getValue());
data.getFieldsMapping().add(idField);
}
return data;
}
private @NotNull QueryConfigEntity buildQueryConfigEntity(String path){
QueryConfigEntity data = new QueryConfigEntity();
data.setName("like");
if ((path.toLowerCase(Locale.ROOT).trim().contains("openaire") || path.toLowerCase(Locale.ROOT).trim().contains("orcid")
|| path.toLowerCase(Locale.ROOT).trim().contains("ror") || path.toLowerCase(Locale.ROOT).trim().contains("fairsharing"))) {
data.setDefaultValue("*");
} else {
data.setDefaultValue("");
}
data.setCases(new ArrayList<>());
QueryCaseConfigEntity caseConfig = new QueryCaseConfigEntity();
caseConfig.setReferenceTypeId(null);
caseConfig.setReferenceTypeSourceKey(null);
caseConfig.setSeparator(null);
caseConfig.setValue("{like}");
caseConfig.setLikePattern(null);
data.getCases().add(caseConfig);
return data;
}
private @NotNull AuthenticationConfigurationEntity buildAuthConfigEntity(AutoCompleteData.AuthAutoCompleteData persist){
AuthenticationConfigurationEntity data = new AuthenticationConfigurationEntity();
if (persist == null) return data;
data.setEnabled(true);
data.setAuthUrl(persist.getUrl());
data.setAuthMethod(this.toReferenceTypeExternalApiHTTPMethodType(persist.getMethod()));
data.setAuthRequestBody(persist.getBody());
data.setType(persist.getType());
data.setAuthTokenPath(persist.getPath());
return data;
}
private ReferenceTypeExternalApiHTTPMethodType toReferenceTypeExternalApiHTTPMethodType(String method){
if (this.conventionService.isNullOrEmpty(method)) {
logger.error("Migrate DescriptionTemplate autocomplete method not set use default GET");
return ReferenceTypeExternalApiHTTPMethodType.GET;
} else if (method.toLowerCase(Locale.ROOT).trim().equals("get")) {
return ReferenceTypeExternalApiHTTPMethodType.GET;
} else if (method.toLowerCase(Locale.ROOT).trim().equals("post")) {
return ReferenceTypeExternalApiHTTPMethodType.POST;
} else {
logger.error("Migrate DescriptionTemplate autocomplete method is invalid " + method);
throw new MyApplicationException("Migrate DescriptionTemplate autocomplete method is invalid " + method);
}
}
}

38
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpBlueprintXmlMigrationService.java
View File

@ -120,7 +120,7 @@ public class DmpBlueprintXmlMigrationService {
if (!this.conventionService.isListNullOrEmpty(itm.getFields())) {
data.setFields(new ArrayList<>());
for (FieldModel fieldModel : itm.getFields()) {
data.getFields().add(this.buildExtraFieldEntity(fieldModel));
data.getFields().add(this.buildFieldEntity(fieldModel));
}
}
@ -153,7 +153,7 @@ public class DmpBlueprintXmlMigrationService {
}
private @NotNull eu.eudat.commons.types.dmpblueprint.FieldEntity buildExtraFieldEntity(FieldModel itm) {
private @NotNull eu.eudat.commons.types.dmpblueprint.FieldEntity buildFieldEntity(FieldModel itm) {
if (itm == null)
return new ExtraFieldEntity();
FieldEntity data;
@ -170,33 +170,39 @@ public class DmpBlueprintXmlMigrationService {
}
dataTyped.setRequired(extraField.getRequired());
data = dataTyped;
data.setCategory(DmpBlueprintFieldCategory.Extra);
} else {
SystemFieldEntity dataTyped = new SystemFieldEntity();
ReferenceTypeFieldEntity referenceTypeFieldEntity = new ReferenceTypeFieldEntity();
SystemField systemField = itm.toSystemField();
switch (systemField.getType()){
case TEXT -> dataTyped.setType(DmpBlueprintSystemFieldType.Text);
case TEXT -> dataTyped.setType(DmpBlueprintSystemFieldType.Title);
case HTML_TEXT -> dataTyped.setType(DmpBlueprintSystemFieldType.Description);
case ACCESS_RIGHTS -> dataTyped.setType(DmpBlueprintSystemFieldType.AccessRights);
case CONTACT -> dataTyped.setType(DmpBlueprintSystemFieldType.Contact);
case FUNDER -> dataTyped.setType(DmpBlueprintSystemFieldType.Funder);
case GRANT -> dataTyped.setType(DmpBlueprintSystemFieldType.Grant);
case HTML_TEXT -> dataTyped.setType(DmpBlueprintSystemFieldType.HtmlText);
case LANGUAGE -> dataTyped.setType(DmpBlueprintSystemFieldType.Language);
case LICENSE -> dataTyped.setType(DmpBlueprintSystemFieldType.License);
case ORGANIZATIONS -> dataTyped.setType(DmpBlueprintSystemFieldType.Organizations);
case PROJECT -> dataTyped.setType(DmpBlueprintSystemFieldType.Project);
case RESEARCHERS -> dataTyped.setType(DmpBlueprintSystemFieldType.Researchers);
case FUNDER -> referenceTypeFieldEntity.setReferenceTypeId(ReferenceTypeIds.Funder);
case GRANT -> referenceTypeFieldEntity.setReferenceTypeId(ReferenceTypeIds.Grants);
case LICENSE -> referenceTypeFieldEntity.setReferenceTypeId(ReferenceTypeIds.License);
case ORGANIZATIONS -> referenceTypeFieldEntity.setReferenceTypeId(ReferenceTypeIds.Organizations);
case PROJECT -> referenceTypeFieldEntity.setReferenceTypeId(ReferenceTypeIds.Project);
case RESEARCHERS -> referenceTypeFieldEntity.setReferenceTypeId(ReferenceTypeIds.Researcher);
default -> throw new MyApplicationException("Type not found " + systemField.getType());
}
dataTyped.setRequired(systemField.isRequired());
data = dataTyped;
referenceTypeFieldEntity.setRequired(systemField.isRequired());
if (referenceTypeFieldEntity.getReferenceTypeId() == null){
data = dataTyped;
data.setCategory(DmpBlueprintFieldCategory.System);
}
else{
data = referenceTypeFieldEntity;
data.setCategory(DmpBlueprintFieldCategory.ReferenceType);
}
}
data.setId(itm.getId());
switch (itm.getCategory()){
case EXTRA -> data.setCategory(DmpBlueprintFieldCategory.Extra);
case SYSTEM -> data.setCategory(DmpBlueprintFieldCategory.System);
default -> throw new MyApplicationException("Type not found " + itm.getCategory());
}
data.setLabel(itm.getLabel());
data.setPlaceholder(itm.getPlaceholder());
data.setDescription(itm.getDescription());

83
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DmpMigrationService.java
View File

@ -7,8 +7,8 @@ import eu.eudat.commons.enums.*;
import eu.eudat.commons.types.dmp.DmpBlueprintValueEntity;
import eu.eudat.commons.types.dmp.DmpContactEntity;
import eu.eudat.commons.types.dmp.DmpPropertiesEntity;
import eu.eudat.commons.types.dmpblueprint.ReferenceTypeFieldEntity;
import eu.eudat.commons.types.dmpblueprint.SectionEntity;
import eu.eudat.commons.types.dmpblueprint.SystemFieldEntity;
import eu.eudat.commons.types.dmpreference.DmpReferenceDataEntity;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
@ -87,11 +87,11 @@ public class DmpMigrationService {
logger.debug("Migrate Dmp " + page * PageSize + " of " + total);
List<DmpBlueprintEntity> dmpBlueprintEntities = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().filter(x-> x.getProfile() != null).map(x-> x.getProfile().getId()).distinct().collect(Collectors.toList())).collectAs(new BaseFieldSet().ensure(DmpBlueprint._definition).ensure(DmpBlueprint._id));
Map<UUID, List<SystemFieldEntity>> systemFieldsByDmpBlueprintId = new HashMap<>();
Map<UUID, List<ReferenceTypeFieldEntity>> referenceTypeFieldsByDmpBlueprintId = new HashMap<>();
for (DmpBlueprintEntity dmpBlueprintEntity : dmpBlueprintEntities) {
eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition =this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition());
List<SystemFieldEntity> systemFieldEntities = definition.getSections().stream().map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.System)).map(x-> (SystemFieldEntity)x).collect(Collectors.toList());
systemFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , systemFieldEntities);
eu.eudat.commons.types.dmpblueprint.DefinitionEntity definition = this.xmlHandlingService.fromXml(eu.eudat.commons.types.dmpblueprint.DefinitionEntity.class, dmpBlueprintEntity.getDefinition());
List<ReferenceTypeFieldEntity> referenceTypeFieldEntities = definition.getSections().stream().filter(x-> x.getFields() != null).map(SectionEntity::getFields).flatMap(List::stream).filter(x-> x.getCategory().equals(DmpBlueprintFieldCategory.ReferenceType)).map(x-> (ReferenceTypeFieldEntity)x).collect(Collectors.toList());
referenceTypeFieldsByDmpBlueprintId.put(dmpBlueprintEntity.getId() , referenceTypeFieldEntities);
}
for (DMP item : items) {
//entityManager.detach(item);
@ -175,11 +175,11 @@ public class DmpMigrationService {
this.entityManager.persist(data);
this.entityManager.flush();
licenseIdByName = this.migrateLicense(item, model, licenseIdByName, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateOrganizations(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateResearchers(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateProjects(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateGrantAndFunder(item, systemFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
licenseIdByName = this.migrateLicense(item, model, licenseIdByName, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateOrganizations(item, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateResearchers(item, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateProjects(item, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
this.migrateGrantAndFunder(item, referenceTypeFieldsByDmpBlueprintId.getOrDefault(data.getBlueprintId(), null));
}
page++;
@ -187,16 +187,16 @@ public class DmpMigrationService {
} while (items != null && !items.isEmpty() && !TestMode);
}
private void migrateOrganizations(DMP item, List<SystemFieldEntity> systemFieldEntities){
private void migrateOrganizations(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
Set<Organisation> organisations = item.getOrganisations();
if (organisations == null || organisations.isEmpty()) return;
if (systemFieldEntities == null) {
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Organizations)).findFirst().orElse(null);
if (systemField == null) {
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Organizations)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed Organizations field not found " + item.getId());
throw new MyApplicationException("Migration failed Organizations field not found " + item.getId());
}
@ -208,22 +208,22 @@ public class DmpMigrationService {
data.setReferenceId(organisation.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
this.entityManager.flush();
}
private void migrateProjects(DMP item, List<SystemFieldEntity> systemFieldEntities){
private void migrateProjects(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
Project project = item.getProject();
if (project == null) return;
if (systemFieldEntities == null) {
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Project)).findFirst().orElse(null);
if (systemField == null) {
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Project)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed Project field not found " + item.getId());
throw new MyApplicationException("Migration failed Project field not found " + item.getId());
}
@ -235,20 +235,20 @@ public class DmpMigrationService {
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setIsActive(IsActive.Active);
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
this.entityManager.persist(data);
this.entityManager.flush();
}
private void migrateResearchers(DMP item, List<SystemFieldEntity> systemFieldEntities){
private void migrateResearchers(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
Set<Researcher> researchers = item.getResearchers();
if (researchers == null || researchers.isEmpty()) return;
if (systemFieldEntities == null) {
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Researchers)).findFirst().orElse(null);
if (systemField == null) {
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Researcher)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed Researchers field not found " + item.getId());
throw new MyApplicationException("Migration failed Researchers field not found " + item.getId());
}
@ -260,23 +260,23 @@ public class DmpMigrationService {
data.setReferenceId(researcher.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
}
this.entityManager.flush();
}
private void migrateGrantAndFunder(DMP item, List<SystemFieldEntity> systemFieldEntities){
private void migrateGrantAndFunder(DMP item, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities){
Grant grant = item.getGrant();
if (grant == null) return;
if (systemFieldEntities == null) {
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Grant)).findFirst().orElse(null);
if (systemField == null) {
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Grants)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed Grant field not found " + item.getId());
throw new MyApplicationException("Migration failed Grant field not found " + item.getId());
}
@ -287,13 +287,12 @@ public class DmpMigrationService {
data.setReferenceId(grant.getId());
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
data.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
data.setIsActive(IsActive.Active);
this.entityManager.persist(data);
if (grant.getFunder() != null){
SystemFieldEntity funderSystemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.Funder)).findFirst().orElse(null);
if (funderSystemField == null) {
ReferenceTypeFieldEntity funderField = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.Funder)).findFirst().orElse(null);
if (funderField == null) {
logger.warn("Migration failed Funder field not found " + item.getId());
throw new MyApplicationException("Migration failed Funder field not found " + item.getId());
}
@ -303,22 +302,22 @@ public class DmpMigrationService {
founder.setReferenceId(grant.getFunder().getId());
founder.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
founder.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
founder.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(funderSystemField)));
founder.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(funderField)));
founder.setIsActive(IsActive.Active);
this.entityManager.persist(founder);
}
this.entityManager.flush();
}
private Map<String, UUID> migrateLicense(DMP item, DataManagementPlan model, Map<String, UUID> licenseIdByName, List<SystemFieldEntity> systemFieldEntities) throws JsonProcessingException {
private Map<String, UUID> migrateLicense(DMP item, DataManagementPlan model, Map<String, UUID> licenseIdByName, List<ReferenceTypeFieldEntity> referenceTypeFieldEntities) throws JsonProcessingException {
if (model.getExtraProperties() != null) {
if (model.getExtraProperties().containsKey("license") && model.getExtraProperties().get("license") != null) {
if (systemFieldEntities == null) {
if (referenceTypeFieldEntities == null) {
logger.warn("Migration failed Blueprint not found " + item.getProfile().getId());
throw new MyApplicationException("Migration failed Blueprint not found " + item.getProfile().getId());
}
SystemFieldEntity systemField = systemFieldEntities.stream().filter(x-> x.getType().equals(DmpBlueprintSystemFieldType.License)).findFirst().orElse(null);
if (systemField == null) {
ReferenceTypeFieldEntity referenceTypeFieldEntity = referenceTypeFieldEntities.stream().filter(x-> x.getReferenceTypeId().equals(ReferenceTypeIds.License)).findFirst().orElse(null);
if (referenceTypeFieldEntity == null) {
logger.warn("Migration failed License field not found " + item.getId());
throw new MyApplicationException("Migration failed License field not found " + item.getId());
}
@ -383,7 +382,7 @@ public class DmpMigrationService {
referenceEntity.setId(UUID.randomUUID());
referenceEntity.setSourceType(ReferenceSourceType.External);
referenceEntity.setSource(this.environment.getProperty("migration.default-license-source", "null"));
referenceEntity.setType(ReferenceType.Licenses);
referenceEntity.setTypeId(ReferenceTypeIds.License);
referenceEntity.setIsActive(IsActive.Active);
definitionEntity.setFields(fields);
referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
@ -398,7 +397,7 @@ public class DmpMigrationService {
dmpReferenceEntity.setReferenceId(licenseIdByName.get(licensePid));
dmpReferenceEntity.setCreatedAt(Instant.now());
dmpReferenceEntity.setUpdatedAt(Instant.now());
dmpReferenceEntity.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(systemField)));
dmpReferenceEntity.setData(this.jsonHandlingService.toJsonSafe(this.buildDmpReferenceDataEntity(referenceTypeFieldEntity)));
dmpReferenceEntity.setIsActive(IsActive.Active);
if (!licenseExists) entityManager.persist(referenceEntity);
@ -410,9 +409,9 @@ public class DmpMigrationService {
return licenseIdByName;
}
private DmpReferenceDataEntity buildDmpReferenceDataEntity(SystemFieldEntity systemField){
private DmpReferenceDataEntity buildDmpReferenceDataEntity(ReferenceTypeFieldEntity referenceTypeFieldEntity){
DmpReferenceDataEntity data = new DmpReferenceDataEntity();
data.setBlueprintFieldId(systemField.getId());
data.setBlueprintFieldId(referenceTypeFieldEntity.getId());
return data;
}

4
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ExternalDatasetMigrationService.java
View File

@ -2,10 +2,8 @@ package eu.old.eudat.migration;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.data.ReferenceEntity;
import eu.old.eudat.data.dao.entities.ExternalDatasetDao;
import eu.old.eudat.data.entities.DMP;
import eu.old.eudat.data.entities.ExternalDataset;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.logging.LoggerService;
@ -56,7 +54,7 @@ public class ExternalDatasetMigrationService {
data.setLabel(item.getLabel());
data.setAbbreviation(item.getAbbreviation());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Datasets);
data.setTypeId(ReferenceTypeIds.Datasets);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId());

11
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/FunderMigrationService.java
View File

@ -1,11 +1,12 @@
package eu.old.eudat.migration;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.data.ReferenceEntity;
import eu.old.eudat.data.dao.entities.FunderDao;
import eu.old.eudat.data.entities.DMP;
import eu.old.eudat.data.entities.Funder;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.logging.LoggerService;
@ -13,6 +14,7 @@ import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Locale;
@ -54,14 +56,15 @@ public class FunderMigrationService {
ReferenceEntity data = new ReferenceEntity();
data.setId(item.getId());
data.setLabel(item.getLabel());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Funder);
data.setIsActive(Funder.Status.fromInteger(item.getStatus()).equals(Funder.Status.ACTIVE) ? IsActive.Active : IsActive.Inactive);
data.setTypeId(ReferenceTypeIds.Funder);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId());
data.setReference(referenceParts[1]);
data.setSource(isInternal? "Internal" : referenceParts[0]);
data.setSourceType(Funder.FunderType.INTERNAL.getValue().equals(item.getType())? ReferenceSourceType.Internal : ReferenceSourceType.External);
this.entityManager.persist(data);
}
this.entityManager.flush();

5
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/GrantMigrationService.java
View File

@ -4,7 +4,6 @@ import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
@ -64,8 +63,8 @@ public class GrantMigrationService {
data.setLabel(item.getLabel());
data.setDescription(item.getDescription());
data.setAbbreviation(item.getAbbreviation());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Grants);
data.setIsActive(Grant.Status.fromInteger(item.getStatus()).equals(Grant.Status.ACTIVE) ? IsActive.Active : IsActive.Inactive);
data.setTypeId(ReferenceTypeIds.Grants);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId());

3
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/OrganizationMigrationService.java
View File

@ -4,7 +4,6 @@ import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
@ -65,7 +64,7 @@ public class OrganizationMigrationService {
data.setLabel(item.getLabel());
data.setAbbreviation(item.getAbbreviation());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Organizations);
data.setTypeId(ReferenceTypeIds.Organizations);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
data.setReference(referenceParts[1]);

5
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ProjectMigrationService.java
View File

@ -4,7 +4,6 @@ import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
@ -65,8 +64,8 @@ public class ProjectMigrationService {
data.setLabel(item.getLabel());
data.setAbbreviation(item.getAbbreviation());
data.setDescription(item.getDescription());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Project);
data.setIsActive(Project.Status.ACTIVE.equals(Project.Status.fromInteger(item.getStatus())) ? IsActive.Active : IsActive.Inactive);
data.setTypeId(ReferenceTypeIds.DataRepositories);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId());

21
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ReferenceTypeIds.java Normal file
View File

@ -0,0 +1,21 @@
package eu.old.eudat.migration;
import java.util.UUID;
public class ReferenceTypeIds {
public static UUID DataRepositories = UUID.fromString("d26b64ba-fda0-4b14-b3ef-9856a492d55a");
public static UUID Datasets = UUID.fromString("a3ce0fb2-d72c-48bb-b322-7401940cb802");
public static UUID Funder = UUID.fromString("538928bb-c7c6-452e-b66d-08e539f5f082");
public static UUID Grants = UUID.fromString("5b9c284f-f041-4995-96cc-fad7ad13289c");
public static UUID Organizations = UUID.fromString("7eeffb98-58fb-4921-82ec-e27f32f8e738");
public static UUID Project = UUID.fromString("3d372db5-a456-45e6-a845-e41e1a8311f8");
public static UUID Registries = UUID.fromString("2beacaad-3223-43ad-ad99-1e5f21328e7b");
public static UUID Researcher = UUID.fromString("5a2112e7-ea99-4cfe-98a1-68665e26726e");
public static UUID Services = UUID.fromString("9ec2000d-95c7-452e-b356-755fc8e2574c");
public static UUID License = UUID.fromString("2baab1e8-561f-4c15-84c3-571b811c52f6");
public static UUID Taxonomy = UUID.fromString("ab7cdd93-bea2-440d-880d-3846dad80b21");
public static UUID Publication = UUID.fromString("51225b6a-86a6-48ac-9192-f15096dbcb8a");
public static UUID Journal = UUID.fromString("8ec7556b-749d-4c4a-a4b9-43d064693795");
public static UUID PubRepositories = UUID.fromString("1e927daa-b856-443f-96da-22f325f7322f");
}

3
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/RegistryMigrationService.java
View File

@ -4,7 +4,6 @@ import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
@ -64,7 +63,7 @@ public class RegistryMigrationService {
data.setLabel(item.getLabel());
data.setAbbreviation(item.getAbbreviation());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Registries);
data.setTypeId(ReferenceTypeIds.Registries);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId());

3
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ResearcherMigrationService.java
View File

@ -4,7 +4,6 @@ import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
@ -64,7 +63,7 @@ public class ResearcherMigrationService {
data.setId(item.getId());
data.setLabel(item.getLabel());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Researcher);
data.setTypeId(ReferenceTypeIds.Researcher);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId());

3
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/ServiceMigrationService.java
View File

@ -4,7 +4,6 @@ import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.convention.ConventionService;
@ -63,7 +62,7 @@ public class ServiceMigrationService {
data.setLabel(item.getLabel());
data.setAbbreviation(item.getAbbreviation());
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Services);
data.setTypeId(ReferenceTypeIds.Services);
data.setCreatedAt(item.getCreated().toInstant());
data.setUpdatedAt(item.getModified().toInstant());
if (item.getCreationUser() != null) data.setCreatedById(item.getCreationUser().getId());

6
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserContactInfoMigrationService.java
View File

@ -1,15 +1,9 @@
package eu.old.eudat.migration;
import eu.eudat.commons.enums.ContactInfoType;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.UserContactInfoEntity;
import eu.old.eudat.data.dao.entities.FunderDao;
import eu.old.eudat.data.dao.entities.UserInfoDao;
import eu.old.eudat.data.entities.Credential;
import eu.old.eudat.data.entities.Funder;
import eu.old.eudat.data.entities.UserInfo;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.logging.LoggerService;

14
dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/UserMigrationService.java
View File

@ -4,21 +4,15 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
import eu.eudat.commons.enums.ReferenceType;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.commons.types.reference.FieldEntity;
import eu.eudat.commons.types.user.AdditionalInfoEntity;
import eu.eudat.data.ReferenceEntity;
import eu.eudat.data.UserEntity;
import eu.eudat.model.Reference;
import eu.eudat.model.User;
import eu.eudat.query.ReferenceQuery;
import eu.eudat.query.UserQuery;
import eu.old.eudat.data.dao.entities.UserInfoDao;
import eu.old.eudat.data.entities.UserInfo;
import eu.old.eudat.elastic.entities.Organization;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.Paging;
@ -28,7 +22,9 @@ import jakarta.persistence.EntityManager;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
@Service
@ -55,7 +51,7 @@ public class UserMigrationService {
int page = 0;
List<UserEntity> items;
List<ReferenceEntity> organizations = this.queryFactory.query(ReferenceQuery.class).types(ReferenceType.Organizations).isActive(IsActive.Active).collect();
List<ReferenceEntity> organizations = this.queryFactory.query(ReferenceQuery.class).typeIds(ReferenceTypeIds.Organizations).isActive(IsActive.Active).collect();
do {
UserQuery userQuery = this.queryFactory.query(UserQuery.class);
@ -110,7 +106,7 @@ public class UserMigrationService {
data.setLabel(organizationName);
data.setAbbreviation(null);
data.setIsActive(IsActive.Active);
data.setType(ReferenceType.Organizations);
data.setTypeId(ReferenceTypeIds.Organizations);
data.setCreatedAt(item.getCreatedAt());
data.setUpdatedAt(item.getUpdatedAt());
data.setReference(referenceParts[1]);

36
dmp-migration-tool/web/src/main/java/eu/old/eudat/publicapi/migration/MigrationController.java
View File

@ -15,6 +15,7 @@ import javax.management.InvalidApplicationException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.io.IOException;
import java.net.URISyntaxException;
@Api(tags = "Migration")
@RestController
@ -41,6 +42,7 @@ public class MigrationController {
private final UserMigrationService userMigrationService;
private final DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService;
private final DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService;
private final DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService;
public MigrationController(
DmpMigrationService dmpMigrationService,
@ -59,8 +61,8 @@ public class MigrationController {
UserContactInfoMigrationService userContactInfoMigrationService,
DmpUserMigrationService dmpUserMigrationService,
UserMigrationService userMigrationService,
DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService,
DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService) {
DmpBlueprintXmlMigrationService dmpBlueprintXmlMigrationService,
DescriptionTemplateXmlMigrationService descriptionTemplateXmlMigrationService, DescriptionTemplateXmlCleanInvalidReferenceTypesService descriptionTemplateXmlCleanInvalidReferenceTypesService) {
this.dmpMigrationService = dmpMigrationService;
this.datasetMigrationService = datasetMigrationService;
this.dmpDatasetProfileMigrationService = dmpDatasetProfileMigrationService;
@ -80,11 +82,12 @@ public class MigrationController {
this.userMigrationService = userMigrationService;
this.dmpBlueprintXmlMigrationService = dmpBlueprintXmlMigrationService;
this.descriptionTemplateXmlMigrationService = descriptionTemplateXmlMigrationService;
this.descriptionTemplateXmlCleanInvalidReferenceTypesService = descriptionTemplateXmlCleanInvalidReferenceTypesService;
}
@GetMapping("all")
@Transactional
public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException, NoSuchFieldException, InvalidApplicationException, TransformerException {
public boolean migrateAll() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException, NoSuchFieldException, InvalidApplicationException, TransformerException, URISyntaxException {
this.dataRepositoryMigrationService.migrate();
this.externalDatasetMigrationService.migrate();
this.funderMigrationService.migrate();
@ -95,21 +98,22 @@ public class MigrationController {
this.researcherMigrationService.migrate();
this.serviceMigrationService.migrate();
this.userContactInfoMigrationService.migrate();
this.userMigrationService.migrate();
//XML recreate
this.dmpBlueprintXmlMigrationService.migrate();
this.descriptionTemplateXmlMigrationService.migrate();
this.dmpMigrationService.migrate();
this.dmpDatasetProfileMigrationService.migrate();
this.datasetMigrationService.migrate();
//this.datasetMigrationService.migrate(); //TODO
this.tagMigrationService.migrate();
this.dmpUserMigrationService.migrate();
this.referenceMigrationService.migrateDatasetReferences();
this.userContactInfoMigrationService.migrate();
this.userMigrationService.migrate();
this.referenceMigrationService.migrateDatasetReferences(); //TODO
//XML recreate
this.dmpBlueprintXmlMigrationService.migrate();
this.descriptionTemplateXmlMigrationService.migrate();
return true;
}
@ -123,12 +127,20 @@ public class MigrationController {
@GetMapping("description-template-xml")
@Transactional
public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException {
public boolean migrateDescriptionTemplate() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
this.descriptionTemplateXmlMigrationService.migrate();
return true;
}
@GetMapping("description-template-xml-clean-invalid-reference-types")
@Transactional
public boolean migrateDescriptionTemplateCleanInvalidReferenceTypes() throws JAXBException, InvalidApplicationException, IOException, ParserConfigurationException, NoSuchFieldException, TransformerException, IllegalAccessException, InstantiationException, SAXException, URISyntaxException {
this.descriptionTemplateXmlCleanInvalidReferenceTypesService.migrate();
return true;
}
@GetMapping("contacts")
@Transactional
public boolean migrateContacts() {