argos/dmp-migration-tool/web/src/main/java/eu/old/eudat/migration/DatasetMigrationService.java

633 lines
39 KiB
Java
Raw Normal View History

2023-11-23 09:26:40 +01:00
package eu.old.eudat.migration;
import eu.eudat.commons.JsonHandlingService;
2023-11-23 14:23:49 +01:00
import eu.eudat.commons.XmlHandlingService;
2024-02-22 17:33:52 +01:00
import eu.eudat.commons.enums.DescriptionStatus;
import eu.eudat.commons.enums.IsActive;
2024-02-23 17:55:35 +01:00
import eu.eudat.commons.enums.ReferenceFieldDataType;
import eu.eudat.commons.enums.ReferenceSourceType;
2024-02-22 17:33:52 +01:00
import eu.eudat.commons.types.description.*;
2024-02-21 17:38:28 +01:00
import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
2024-02-23 17:55:35 +01:00
import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
2023-11-23 14:23:49 +01:00
import eu.eudat.commons.types.dmpblueprint.DefinitionEntity;
2024-02-23 17:55:35 +01:00
import eu.eudat.commons.types.externalfetcher.ExternalFetcherBaseSourceConfigurationEntity;
import eu.eudat.commons.types.referencetype.ReferenceTypeDefinitionEntity;
2024-02-22 17:33:52 +01:00
import eu.eudat.convention.ConventionService;
2024-02-21 17:38:28 +01:00
import eu.eudat.data.*;
2023-11-23 17:09:17 +01:00
import eu.eudat.model.Dmp;
2024-02-23 17:55:35 +01:00
import eu.eudat.model.Reference;
import eu.eudat.model.ReferenceType;
import eu.eudat.query.*;
2023-11-23 14:23:49 +01:00
import eu.old.eudat.data.dao.entities.DatasetDao;
2023-11-23 09:26:40 +01:00
import eu.old.eudat.data.entities.Dataset;
import eu.old.eudat.logic.services.operations.DatabaseRepository;
2023-11-23 14:23:49 +01:00
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
2023-11-23 17:09:17 +01:00
import gr.cite.tools.fieldset.BaseFieldSet;
2023-11-23 14:23:49 +01:00
import gr.cite.tools.logging.LoggerService;
2023-11-23 09:26:40 +01:00
import jakarta.persistence.EntityManager;
2023-11-23 14:23:49 +01:00
import jakarta.xml.bind.JAXBException;
2023-11-23 17:09:17 +01:00
import org.json.JSONObject;
2023-11-23 14:23:49 +01:00
import org.slf4j.LoggerFactory;
2023-11-23 09:26:40 +01:00
import org.springframework.stereotype.Service;
2023-11-23 14:23:49 +01:00
import org.xml.sax.SAXException;
2023-11-23 09:26:40 +01:00
2023-11-23 14:23:49 +01:00
import javax.xml.parsers.ParserConfigurationException;
import java.io.IOException;
import java.time.Instant;
2024-02-22 17:33:52 +01:00
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.time.format.DateTimeParseException;
2023-11-23 14:23:49 +01:00
import java.util.*;
2023-11-23 17:09:17 +01:00
import java.util.stream.Collectors;
2023-11-23 09:26:40 +01:00
@Service
public class DatasetMigrationService {
2023-11-23 14:23:49 +01:00
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DmpDatasetProfileMigrationService.class));
2023-11-23 09:26:40 +01:00
private final DatabaseRepository databaseRepository;
private final JsonHandlingService jsonHandlingService;
2023-11-23 14:23:49 +01:00
private final QueryFactory queryFactory;
private final XmlHandlingService xmlHandlingService;
private static final int PageSize = 500;
private static final boolean TestMode = false;
2023-11-23 09:26:40 +01:00
private final EntityManager entityManager;
2024-02-22 17:33:52 +01:00
private final ConventionService conventionService;
2023-11-23 09:26:40 +01:00
2024-02-22 17:33:52 +01:00
public DatasetMigrationService(DatabaseRepository databaseRepository, JsonHandlingService jsonHandlingService, QueryFactory queryFactory, XmlHandlingService xmlHandlingService, EntityManager entityManager, ConventionService conventionService) {
2023-11-23 09:26:40 +01:00
this.databaseRepository = databaseRepository;
this.jsonHandlingService = jsonHandlingService;
2023-11-23 14:23:49 +01:00
this.queryFactory = queryFactory;
this.xmlHandlingService = xmlHandlingService;
2023-11-23 09:26:40 +01:00
this.entityManager = entityManager;
2024-02-22 17:33:52 +01:00
this.conventionService = conventionService;
2023-11-23 09:26:40 +01:00
}
2023-11-23 14:23:49 +01:00
public void migrate() throws IOException, JAXBException, ParserConfigurationException, InstantiationException, IllegalAccessException, SAXException {
DatasetDao datasetDao = databaseRepository.getDatasetDao();
long total = datasetDao.asQueryable().count();
logger.debug("Migrate Dataset Total : " + total);
int page = 0;
2024-02-23 17:55:35 +01:00
Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap = new HashMap<>();
List<ReferenceTypeEntity> referenceTypes = this.queryFactory.query(ReferenceTypeQuery.class).collectAs(new BaseFieldSet().ensure(ReferenceType._id).ensure(ReferenceType._definition));
for (ReferenceTypeEntity referenceType : referenceTypes){
referenceTypeDefinitionEntityMap.put(referenceType.getId(), this.xmlHandlingService.fromXml(ReferenceTypeDefinitionEntity.class, referenceType.getDefinition()));
}
Map<ReferenceKey, ReferenceEntity> referenceMap = new HashMap<>();
List<ReferenceEntity> references = this.queryFactory.query(ReferenceQuery.class).collectAs(new BaseFieldSet().ensure(Reference._id).ensure(Reference._source).ensure(Reference._type).ensure(Reference._reference));
for (ReferenceEntity reference : references){
if (reference.getCreatedById() == null) referenceMap.put(new ReferenceKey(reference), reference);
}
2023-11-23 14:23:49 +01:00
List<Dataset> items;
do {
2023-11-23 17:09:17 +01:00
items = datasetDao.asQueryable().orderBy((builder, root) -> builder.asc(root.get("created"))).orderBy((builder, root) -> builder.asc(root.get("ID"))).skip(page * PageSize).take(PageSize).toList();
2023-11-23 14:23:49 +01:00
if (items != null && !items.isEmpty()) {
logger.debug("Migrate Dataset " + page * PageSize + " of " + total);
2023-11-23 17:09:17 +01:00
List<DmpBlueprintEntity> dmpBlueprints = this.queryFactory.query(DmpBlueprintQuery.class).ids(items.stream().map(x-> x.getDmp().getProfile().getId()).distinct().toList()).collect();
Map<UUID, DefinitionEntity> dmpBlueprintsMap = new HashMap<>();
for (DmpBlueprintEntity dmpBlueprint : dmpBlueprints) {
DefinitionEntity definitionEntity = this.xmlHandlingService.fromXml(DefinitionEntity.class, dmpBlueprint.getDefinition());
dmpBlueprintsMap.put(dmpBlueprint.getId(), definitionEntity);
}
2024-02-21 17:38:28 +01:00
List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities = this.queryFactory.query(DmpDescriptionTemplateQuery.class).dmpIds(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collect();
List<DescriptionTemplateEntity> descriptionTemplates = this.queryFactory.query(DescriptionTemplateQuery.class).ids(items.stream().map(x-> x.getProfile().getId()).distinct().toList()).collect();
2023-11-23 17:09:17 +01:00
Map<UUID, UUID> creatorsByDmp = this.queryFactory.query(DmpQuery.class).ids(items.stream().map(x-> x.getDmp().getId()).distinct().toList()).collectAs(new BaseFieldSet().ensure(Dmp._id).ensure(Dmp._creator)).stream().collect(Collectors.toMap(DmpEntity::getId, DmpEntity::getCreatorId));;
2024-02-21 17:38:28 +01:00
Map<UUID, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity> descriptionTemplateDefinitionMap = new HashMap<>();
for (DescriptionTemplateEntity descriptionTemplateEntity : descriptionTemplates){
descriptionTemplateDefinitionMap.put(descriptionTemplateEntity.getId(), this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition()));
}
2023-11-23 14:23:49 +01:00
for (Dataset item : items) {
2023-11-23 17:09:17 +01:00
DefinitionEntity definition = dmpBlueprintsMap.getOrDefault(item.getDmp().getProfile().getId(), null);
2023-11-29 15:56:25 +01:00
if (definition == null || definition.getSections() == null || definition.getSections().size() <= item.getDmpSectionIndex()) {
2023-11-23 17:09:17 +01:00
logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}
UUID sectionId = definition.getSections().get(item.getDmpSectionIndex()).getId();
if (sectionId == null) {
logger.error("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found section id for section " + item.getDmpSectionIndex());
}
2024-02-21 17:38:28 +01:00
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = this.getOrCreateDmpDescriptionTemplateEntity(item, sectionId, dmpDescriptionTemplateEntities);
2023-11-23 17:09:17 +01:00
if (itemDescriptionTemplates.size() > 1) {
logger.error("Migrate Dataset " + item.getId() + " multiple DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
//TODO CLEAN Duplicates dmp, DescriptionTemplate, SectionId
}
2023-11-23 14:23:49 +01:00
DescriptionEntity data = new DescriptionEntity();
2023-11-23 17:09:17 +01:00
data.setId(item.getId());
data.setDescription(item.getDescription());
if (item.getCreator() != null) {
data.setCreatedById(item.getCreator().getId());
} else {
data.setCreatedById(creatorsByDmp.getOrDefault(item.getDmp().getId(), null));
}
data.setDmpId(item.getDmp().getId());
data.setLabel(item.getLabel());
2024-02-21 17:38:28 +01:00
data.setDmpDescriptionTemplateId(itemDescriptionTemplates.getFirst().getId());
2023-11-24 16:18:28 +01:00
data.setDescriptionTemplateId(item.getProfile().getId());
2023-11-23 17:09:17 +01:00
data.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
data.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
if (item.getFinalizedAt() != null)
data.setFinalizedAt(item.getFinalizedAt().toInstant());
if (item.getStatus() == 99) {
2023-11-23 14:23:49 +01:00
data.setIsActive(IsActive.Inactive);
data.setStatus(DescriptionStatus.Canceled);
} else {
data.setIsActive(IsActive.Active);
2023-11-23 17:09:17 +01:00
data.setStatus(DescriptionStatus.of(item.getStatus()));
}
2024-02-21 17:38:28 +01:00
eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity = descriptionTemplateDefinitionMap.getOrDefault(item.getProfile().getId(), null);
2024-02-23 17:55:35 +01:00
data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(item, descriptionTemplateDefinitionEntity, referenceTypeDefinitionEntityMap, referenceMap)));
2023-11-23 17:09:17 +01:00
2023-11-24 16:18:28 +01:00
if (data.getCreatedById() == null){
2023-11-23 17:09:17 +01:00
logger.error("Migration skipped creator not found " + item.getId());
throw new MyApplicationException("Migration skipped creator not found " + item.getId());
2023-11-23 14:23:49 +01:00
}
2024-02-22 17:33:52 +01:00
2023-11-23 14:23:49 +01:00
this.entityManager.persist(data);
this.entityManager.flush();
}
page++;
2023-11-23 09:26:40 +01:00
}
2023-11-23 14:23:49 +01:00
} while (items != null && !items.isEmpty() && !TestMode);
2024-02-22 17:33:52 +01:00
throw new MyApplicationException("");
}
2024-02-21 17:38:28 +01:00
private List<DmpDescriptionTemplateEntity> getOrCreateDmpDescriptionTemplateEntity(Dataset item, UUID sectionId, List<DmpDescriptionTemplateEntity> dmpDescriptionTemplateEntities){
List<DmpDescriptionTemplateEntity> itemDescriptionTemplates = dmpDescriptionTemplateEntities.stream().filter(x-> x.getDescriptionTemplateGroupId().equals(item.getProfile().getGroupId()) && x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId)).toList();
if (itemDescriptionTemplates.isEmpty()) {
logger.error("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
2024-02-22 17:33:52 +01:00
if (dmpDescriptionTemplateEntities.stream().anyMatch(x -> x.getDmpId().equals(item.getDmp().getId()) && x.getSectionId().equals(sectionId))) {
2024-02-21 17:38:28 +01:00
DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = new DmpDescriptionTemplateEntity();
dmpDescriptionTemplateEntity.setId(UUID.randomUUID());
dmpDescriptionTemplateEntity.setDescriptionTemplateGroupId(item.getProfile().getGroupId());
dmpDescriptionTemplateEntity.setDmpId(item.getDmp().getId());
dmpDescriptionTemplateEntity.setCreatedAt(item.getCreated() != null ? item.getCreated().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setUpdatedAt(item.getModified() != null ? item.getModified().toInstant() : Instant.now());
dmpDescriptionTemplateEntity.setSectionId(sectionId);
dmpDescriptionTemplateEntity.setIsActive(IsActive.Active);
this.entityManager.persist(dmpDescriptionTemplateEntity);
itemDescriptionTemplates = List.of(dmpDescriptionTemplateEntity);
} else {
throw new MyApplicationException("Migrate Dataset " + item.getId() + " cannot found DmpDescriptionTemplateEntity for section " + item.getDmpSectionIndex());
}
}
return itemDescriptionTemplates;
}
2024-02-23 17:55:35 +01:00
private PropertyDefinitionEntity buildPropertyDefinitionEntity(Dataset item, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinitionEntity, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap) {
2024-02-22 17:33:52 +01:00
if (this.conventionService.isNullOrEmpty(item.getProperties())) return null;
2024-02-21 17:38:28 +01:00
JSONObject jObject = new JSONObject(item.getProperties());
Map<String, Object> properties = jObject.toMap();
PropertyDefinitionEntity propertyDefinitionEntity = new PropertyDefinitionEntity();
2024-02-22 17:33:52 +01:00
propertyDefinitionEntity.setFieldSets(new HashMap<>());
2024-02-21 17:38:28 +01:00
List<FieldSetEntity> fieldSetEntities = descriptionTemplateDefinitionEntity.getAllFieldSets();
2024-02-22 17:33:52 +01:00
for (String key : properties.keySet()) {
if (key.toLowerCase(Locale.ROOT).trim().startsWith("commentfieldvalue")){
if (properties.get(key) != null && this.conventionService.isNullOrEmpty(properties.get(key).toString())) { continue; }
FieldSetEntity currentFieldSet = fieldSetEntities.stream().filter(x-> x.getId().equals(key.trim().substring("commentfieldvalue".length()))).findFirst().orElse(null);
if (currentFieldSet == null) {
logger.error("Field set " + key + " not found for comment " + properties.get(key).toString());
continue;
}
this.addSimpleCommentField(propertyDefinitionEntity, currentFieldSet, properties.get(key).toString());
} else {
if (!key.toLowerCase(Locale.ROOT).trim().startsWith("multiple_")) {
FieldSetEntity currentFieldSet = null;
eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = null;
for (FieldSetEntity fieldSetEntity : fieldSetEntities) {
List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> fieldEntities = fieldSetEntity.getFieldById(key.trim());
if (!fieldEntities.isEmpty()) {
currentFieldSet = fieldSetEntity;
currentField = fieldEntities.getFirst();
}
}
if (currentField == null) {
logger.error("Field set not found for field " + key + " " + properties.get(key).toString());
continue;
}
2024-02-23 17:55:35 +01:00
this.addSimpleField(propertyDefinitionEntity, currentFieldSet, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
2024-02-22 17:33:52 +01:00
}
}
}
Map<String, Integer> groupOrdinalMapping = new HashMap<>();
for (String key : properties.keySet()) {
if (key.trim().toLowerCase(Locale.ROOT).startsWith("multiple_")) {
String newKey = key.trim().substring("multiple_".length());
String[] keyParts = newKey.split("_", 4);
String fieldSetId = "";
String groupId = "";
int ordinal = 0;
String fieldId = "";
try {
if (keyParts.length == 4) {
fieldSetId = keyParts[0].trim();
groupId = keyParts[1].trim();
ordinal = Integer.parseInt(keyParts[2].trim());
fieldId = keyParts[3].trim();
} else if (keyParts.length == 3) {
fieldSetId = keyParts[0].trim();
groupId = keyParts[1].trim();
fieldId = keyParts[2].trim();
} else {
logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString());
continue;
//throw new MyApplicationException("Invalid multiple key " + key);
}
} catch (Exception e){
logger.error("Field group key has invalid format " + key + " " + properties.get(key).toString());
}
eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField = descriptionTemplateDefinitionEntity.getFieldById(fieldId).stream().findFirst().orElse(null);
if (currentField == null) {
logger.error("Field set not found for field " + key + " " + properties.get(key).toString());
continue;
}
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(fieldSetId, null);
if (ordinal == 0) {
if (propertyDefinitionFieldSetEntity != null ) {
if (groupOrdinalMapping.containsKey(groupId + fieldSetId)){
ordinal = groupOrdinalMapping.get(groupId + fieldSetId);
} else {
if (propertyDefinitionFieldSetEntity.getItems().stream().anyMatch(x-> x.getOrdinal() == 0 && !x.getFields().isEmpty())) {
ordinal = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> !x.getFields().isEmpty()).map(PropertyDefinitionFieldSetItemEntity::getOrdinal).max(Comparator.comparing(x -> x)).orElse(0) + 1;
int finalOrdinal = ordinal;
if (propertyDefinitionFieldSetEntity.getItems().stream().anyMatch(x -> x.getOrdinal() == finalOrdinal && !x.getFields().isEmpty()))
throw new MyApplicationException("Invalid multiple key " + key);
logger.error("Ordinal of group set to " + ordinal + " " + key);
}
}
} else {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
}
}
if (!groupOrdinalMapping.containsKey(groupId + fieldSetId)){
groupOrdinalMapping.put(groupId + fieldSetId, ordinal);
} else {
if (groupOrdinalMapping.get(groupId + fieldSetId) != ordinal) throw new MyApplicationException("Invalid multiple key ordinal " + key);
}
if (propertyDefinitionFieldSetEntity == null) throw new MyApplicationException("Invalid multiple key group " + key);
2024-02-23 17:55:35 +01:00
this.addMultipleField(propertyDefinitionFieldSetEntity, ordinal, currentField, properties, referenceTypeDefinitionEntityMap, referenceMap);
2024-02-22 17:33:52 +01:00
}
}
2024-02-21 17:38:28 +01:00
return propertyDefinitionEntity;
2024-02-22 17:33:52 +01:00
}
2024-02-23 17:55:35 +01:00
private void addMultipleField(PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity, int ordinal, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
2024-02-22 17:33:52 +01:00
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().stream().filter(x-> x.getOrdinal() == ordinal).findFirst().orElse(null);
if (propertyDefinitionFieldSetItemEntity == null){
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
propertyDefinitionFieldSetItemEntity.setOrdinal(ordinal);
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
}
2024-02-23 17:55:35 +01:00
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
2024-02-22 17:33:52 +01:00
}
2024-02-23 17:55:35 +01:00
private void addSimpleField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
2024-02-22 17:33:52 +01:00
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity);
}
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null;
if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
propertyDefinitionFieldSetItemEntity.setOrdinal(0);
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
} else {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
}
2024-02-23 17:55:35 +01:00
propertyDefinitionFieldSetItemEntity.getFields().put(currentField.getId(), this.buildField(currentField, properties, referenceTypeDefinitionEntityMap, referenceMap));
2024-02-22 17:33:52 +01:00
}
2024-02-21 17:38:28 +01:00
2024-02-22 17:33:52 +01:00
private void addSimpleCommentField(PropertyDefinitionEntity propertyDefinitionEntity, FieldSetEntity currentFieldSet, String comment){
PropertyDefinitionFieldSetEntity propertyDefinitionFieldSetEntity = propertyDefinitionEntity.getFieldSets().getOrDefault(currentFieldSet.getId(), null);
if (propertyDefinitionFieldSetEntity == null) {
propertyDefinitionFieldSetEntity = new PropertyDefinitionFieldSetEntity();
propertyDefinitionFieldSetEntity.setItems(new ArrayList<>());
propertyDefinitionEntity.getFieldSets().put(currentFieldSet.getId(), propertyDefinitionFieldSetEntity);
}
PropertyDefinitionFieldSetItemEntity propertyDefinitionFieldSetItemEntity = null;
if (this.conventionService.isListNullOrEmpty(propertyDefinitionFieldSetEntity.getItems())){
propertyDefinitionFieldSetItemEntity = new PropertyDefinitionFieldSetItemEntity();
propertyDefinitionFieldSetItemEntity.setFields(new HashMap<>());
propertyDefinitionFieldSetItemEntity.setOrdinal(0);
propertyDefinitionFieldSetEntity.getItems().add(propertyDefinitionFieldSetItemEntity);
} else {
propertyDefinitionFieldSetItemEntity = propertyDefinitionFieldSetEntity.getItems().getFirst();
}
propertyDefinitionFieldSetItemEntity.setComment(comment);
2024-02-21 17:38:28 +01:00
}
2024-02-22 17:33:52 +01:00
2024-02-23 17:55:35 +01:00
private FieldEntity buildField(eu.eudat.commons.types.descriptiontemplate.FieldEntity currentField, Map<String, Object> properties, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, Map<ReferenceKey, ReferenceEntity> referenceMap){
2024-02-22 17:33:52 +01:00
FieldEntity fieldEntity = new FieldEntity();
String textValue = properties.get(currentField.getId()) != null ? properties.get(currentField.getId()).toString() : null;
if (textValue == null || textValue.isEmpty()) return fieldEntity;
switch (currentField.getData().getFieldType()){
case FREE_TEXT, TEXT_AREA, RICH_TEXT_AREA, RADIO_BOX -> fieldEntity.setTextValue(textValue.trim());
case CHECK_BOX, BOOLEAN_DECISION -> fieldEntity.setTextValue(textValue.trim().toLowerCase(Locale.ROOT));
case DATE_PICKER -> {
Instant instant = null;
if(!this.conventionService.isNullOrEmpty(textValue)) {
try {
instant = Instant.parse(textValue);
} catch (DateTimeParseException ex) {
instant = LocalDate.parse(textValue).atStartOfDay().toInstant(ZoneOffset.UTC);
}
}
fieldEntity.setDateValue(instant);
}
case SELECT -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
String[] valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, textValue);
if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(String[].class, this.cleanAsObjectString(textValue));
fieldEntity.setTextListValue(valuesParsed == null ? List.of(textValue) : Arrays.stream(valuesParsed).toList());
}
}
case DATASET_IDENTIFIER, VALIDATION -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
ExternalIdentifierEntity externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, textValue);
if (externalIdentifierEntity == null) externalIdentifierEntity = this.jsonHandlingService.fromJsonSafe(ExternalIdentifierEntity.class, this.cleanAsObjectString(textValue));
if (externalIdentifierEntity == null) throw new MyApplicationException("Could not parse dataset External Identifier : " + textValue);
fieldEntity.setExternalIdentifier(externalIdentifierEntity);
}
}
case UPLOAD -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Map<String, String> valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, textValue);
if (valuesParsed == null) {
valuesParsed = (Map<String, String>) properties.get(currentField.getId());
}
if (valuesParsed == null) valuesParsed = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue));
if (valuesParsed == null) throw new MyApplicationException("Could not parse upload : " + textValue);
String id = valuesParsed.getOrDefault("id", null);
if (id == null) throw new MyApplicationException("Could not parse upload : " + textValue);
fieldEntity.setTextValue(UUID.fromString(id).toString());
}
}
case CURRENCY -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Currency currency = this.jsonHandlingService.fromJsonSafe(Currency.class, textValue);
if (currency == null) currency = this.jsonHandlingService.fromJsonSafe(Currency.class, this.cleanAsObjectString(textValue));
2024-02-23 17:55:35 +01:00
if (currency == null && textValue.toUpperCase(Locale.ROOT).contains("EUR")) {
currency = new Currency();
currency.setName("Euro");
currency.setValue("EUR");
}
if (currency == null && textValue.toUpperCase(Locale.ROOT).contains("US DOLLAR")) {
currency = new Currency();
currency.setName("US DOLLAR");
currency.setValue("USD");
}
if (currency == null) throw new MyApplicationException("Could not parse Currency : " + textValue);
2024-02-22 17:33:52 +01:00
//TODO: {"name":"Euro","value":"EUR"} what we want to keep ?
2024-02-23 17:55:35 +01:00
fieldEntity.setTextValue(currency.getName());
2024-02-22 17:33:52 +01:00
}
}
case TAGS -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Tag[] tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, textValue);
if (tags == null) tags = this.jsonHandlingService.fromJsonSafe(Tag[].class, this.cleanAsObjectString(textValue));
if (tags == null) {
Tag tag = this.jsonHandlingService.fromJsonSafe(Tag.class, textValue);
if (tag == null) tag = this.jsonHandlingService.fromJsonSafe(Tag.class, this.cleanAsObjectString(textValue));
if (tag != null) tags = List.of(tag).toArray(Tag[]::new);
}
2024-02-23 17:55:35 +01:00
if (tags == null) {
//throw new MyApplicationException("Could not parse tag : " + textValue); //TODO
2024-02-22 17:33:52 +01:00
logger.error("Could not parse tag : " + textValue);
2024-02-23 17:55:35 +01:00
break;
2024-02-22 17:33:52 +01:00
}
2024-02-23 17:55:35 +01:00
fieldEntity.setTextListValue(Arrays.stream(tags).toList().stream().map(Tag::getName).toList());
2024-02-22 17:33:52 +01:00
}
}
2024-02-23 17:55:35 +01:00
case INTERNAL_ENTRIES_DMPS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description");
case INTERNAL_ENTRIES_DESCRIPTIONS -> throw new MyApplicationException("Found INTERNAL_ENTRIES_DMPS into description");
case REFERENCE_TYPES -> {
if(!this.conventionService.isNullOrEmpty(textValue)) {
Map<String, Object>[] references = this.jsonHandlingService.fromJsonSafe(Map[].class, textValue);
if (references == null) references = this.jsonHandlingService.fromJsonSafe(Map[].class, this.cleanAsObjectString(textValue));
if (references == null) {
Map<String, Object> reference = this.jsonHandlingService.fromJsonSafe(Map.class, textValue);
if (reference == null) reference = this.jsonHandlingService.fromJsonSafe(Map.class, this.cleanAsObjectString(textValue));
if (reference != null) references = List.of(reference).toArray(Map[]::new);
}
if (references == null){
logger.warn("Could not parse reference : " + textValue);
//throw new MyApplicationException("Could not parse reference : " + textValue);
break;
}
ReferenceTypeDataEntity referenceTypeDataEntity = (ReferenceTypeDataEntity)currentField.getData();
if (referenceTypeDataEntity == null) throw new MyApplicationException("Could not parse description template reference : " + textValue);
fieldEntity.setTextListValue(new ArrayList<>());
for (Map<String, Object> item : references){
ReferenceEntity referenceEntity = this.geReferenceEntity(item, referenceTypeDefinitionEntityMap, referenceTypeDataEntity, textValue, referenceMap);
fieldEntity.getTextListValue().add(referenceEntity.getId().toString());
}
}
}
2024-02-22 17:33:52 +01:00
}
return fieldEntity;
}
2024-02-23 17:55:35 +01:00
private ReferenceEntity geReferenceEntity(Map<String, Object> item, Map<UUID, ReferenceTypeDefinitionEntity> referenceTypeDefinitionEntityMap, ReferenceTypeDataEntity referenceTypeDataEntity, String textValue, Map<ReferenceKey, ReferenceEntity> referenceMap){
ReferenceTypeDefinitionEntity referenceTypeDefinition = referenceTypeDefinitionEntityMap.getOrDefault(referenceTypeDataEntity.getReferenceTypeId(), null);
if (referenceTypeDefinition == null) throw new MyApplicationException("Could not find reference type for reference : " + textValue);
if (this.conventionService.isListNullOrEmpty(referenceTypeDefinition.getSources())) throw new MyApplicationException("Could not find reference source for reference : " + textValue);
ReferenceEntity data = new ReferenceEntity();
if (item.containsKey("name")) data.setLabel((String) item.get("name"));
if (this.conventionService.isNullOrEmpty(data.getLabel()) && item.containsKey("label")) data.setLabel((String) item.get("label"));
if (item.containsKey("abbreviation")) data.setAbbreviation((String) item.get("abbreviation"));
if (item.containsKey("id")) data.setReference((String)item.get("id"));
if (item.containsKey("reference")) data.setReference((String)item.get("reference"));
if (this.conventionService.isNullOrEmpty(data.getReference()) && item.containsKey("pid")) data.setReference((String)item.get("pid"));
data.setId(UUID.randomUUID());
data.setCreatedAt(Instant.now());
data.setUpdatedAt(Instant.now());
data.setSourceType(ReferenceSourceType.External);
data.setTypeId(referenceTypeDataEntity.getReferenceTypeId());
data.setIsActive(IsActive.Active);
eu.eudat.commons.types.reference.DefinitionEntity definitionEntity = new eu.eudat.commons.types.reference.DefinitionEntity();
definitionEntity.setFields(new ArrayList<>());
for (Map.Entry<String, Object> entries : item.entrySet()){
if (entries.getValue() != null && !entries.getKey().equals("name") && !entries.getKey().equals("label")&& !entries.getKey().equals("abbreviation")
&& !entries.getKey().equals("id") && !entries.getKey().equals("pid")&& !entries.getKey().equals("source")) {
eu.eudat.commons.types.reference.FieldEntity field = new eu.eudat.commons.types.reference.FieldEntity();
field.setCode(entries.getKey());
field.setDataType(ReferenceFieldDataType.Text);
field.setValue(entries.getValue().toString());
definitionEntity.getFields().add(field);
}
}
if (!this.conventionService.isListNullOrEmpty(definitionEntity.getFields())) data.setDefinition(this.xmlHandlingService.toXmlSafe(definitionEntity));
if (this.conventionService.isNullOrEmpty(data.getLabel())) data.setLabel(data.getReference());
if (referenceTypeDefinition.getSources().size() == 1) data.setSource(referenceTypeDefinition.getSources().getFirst().getKey());
else {
String key = referenceTypeDefinition.getSources().stream().filter(x-> x.getKey().equalsIgnoreCase((String) item.getOrDefault("source", null))).map(ExternalFetcherBaseSourceConfigurationEntity::getKey).findFirst().orElse(null);
if (this.conventionService.isNullOrEmpty(key)) {
logger.warn("Select first source for reference : " + textValue);
data.setSourceType(ReferenceSourceType.Internal);
data.setSource(referenceTypeDefinition.getSources().getFirst().getKey());
//throw new MyApplicationException("Could not find reference source for reference : " + textValue); //TODO
} else {
data.setSource(key);
}
}
if (this.conventionService.isNullOrEmpty(data.getReference())) {
data.setReference(this.queryFactory.query(ReferenceQuery.class).sources(data.getSource()).like(data.getLabel()).typeIds(data.getTypeId()).collectAs(new BaseFieldSet().ensure(Reference._reference, Reference._label))
.stream().filter(x-> x.getLabel().equals(data.getLabel())).map(ReferenceEntity::getReference).findFirst().orElse(null));//TODO
}
if (this.conventionService.isNullOrEmpty(data.getReference())) {
logger.warn("Local reference created with random id" + textValue);
data.setReference(data.getId().toString().replace("-", ""));
data.setSourceType(ReferenceSourceType.Internal); //TODO
}
ReferenceEntity existingEntity = referenceMap.getOrDefault(new ReferenceKey(data), null);
if(existingEntity == null){
referenceMap.put(new ReferenceKey(data), data);
this.entityManager.persist(data);
return data;
} else {
return existingEntity;
}
}
2024-02-22 17:33:52 +01:00
private String cleanAsObjectString(String value){
value = value.trim().replace("\\", "");
2024-02-23 17:55:35 +01:00
value = value.trim().replace("[\"{", "[{");
value = value.trim().replace("}\"]", "}]");
value = value.trim().replace("}\", \"{", "}, {");
value = value.trim().replace("}\",\"{", "},{");
2024-02-22 17:33:52 +01:00
while (!this.conventionService.isNullOrEmpty(value) && value.startsWith("\"")) value = value.substring(1);
while (!this.conventionService.isNullOrEmpty(value) && value.endsWith("\"")) value = value.substring(0, value.length() - 1);
return value;
}
public static class Currency{
private String name;
private String value;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getValue() {
return value;
}
2023-11-23 09:26:40 +01:00
2024-02-22 17:33:52 +01:00
public void setValue(String value) {
this.value = value;
}
}
public static class Tag{
private String id;
private String name;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
}
2024-02-23 17:55:35 +01:00
public class ReferenceKey {
private final UUID type;
private final String source;
private final String reference;
private final int hashCode;
public ReferenceKey(ReferenceEntity reference) {
this.type = reference.getTypeId();
this.source = reference.getSource().trim();
this.reference = reference.getReference().trim();
hashCode = Objects.hash(this.type, this.source, this.reference);
}
public UUID getType() {
return type;
}
public String getSource() {
return source;
}
public String getReference() {
return reference;
}
@Override
public boolean equals(Object o) {
if (this == o)
return true;
if (o == null || getClass() != o.getClass())
return false;
ReferenceKey that = (ReferenceKey) o;
return Objects.equals(type, that.getType()) && Objects.equals(source, that.getSource()) && Objects.equals(reference, that.getReference());
}
@Override
public int hashCode() {
return this.hashCode;
}
}
2023-11-23 09:26:40 +01:00
}