argos/dmp-backend/core/src/main/java/eu/eudat/service/description/DescriptionServiceImpl.java

package eu.eudat.service.description;

import eu.eudat.authorization.AffiliatedResource;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.authorization.authorizationcontentresolver.AuthorizationContentResolver;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.*;
import eu.eudat.commons.enums.notification.NotificationContactType;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.commons.types.description.*;
import eu.eudat.commons.types.descriptionreference.DescriptionReferenceDataEntity;
import eu.eudat.commons.types.descriptiontemplate.FieldSetEntity;
import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
import eu.eudat.commons.types.descriptiontemplate.fielddata.UploadDataEntity;
import eu.eudat.commons.types.notification.*;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.configurations.notification.NotificationProperties;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.*;
import eu.eudat.errorcode.ErrorThesaurusProperties;
import eu.eudat.event.DescriptionTouchedEvent;
import eu.eudat.event.EventBroker;
import eu.eudat.integrationevent.outbox.annotationentityremoval.AnnotationEntityRemovalIntegrationEventHandler;
import eu.eudat.integrationevent.outbox.annotationentitytouch.AnnotationEntityTouchedIntegrationEventHandler;
import eu.eudat.integrationevent.outbox.descriptiontouched.DescriptionTouchedIntegrationEventHandler;
import eu.eudat.integrationevent.outbox.notification.NotifyIntegrationEvent;
import eu.eudat.integrationevent.outbox.notification.NotifyIntegrationEventHandler;
import eu.eudat.model.*;
import eu.eudat.model.builder.DescriptionBuilder;
import eu.eudat.model.deleter.DescriptionDeleter;
import eu.eudat.model.deleter.DescriptionReferenceDeleter;
import eu.eudat.model.deleter.DescriptionTagDeleter;
import eu.eudat.model.file.FileEnvelope;
import eu.eudat.model.persist.*;
import eu.eudat.model.persist.descriptionproperties.*;
import eu.eudat.model.persist.descriptionreference.DescriptionReferenceDataPersist;
import eu.eudat.model.persist.referencedefinition.DefinitionPersist;
import eu.eudat.query.*;
import eu.eudat.service.elastic.ElasticService;
import eu.eudat.service.storage.StorageFileProperties;
import eu.eudat.service.storage.StorageFileService;
import eu.eudat.service.transformer.FileTransformerService;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.*;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.fieldset.FieldSet;
import gr.cite.tools.logging.LoggerService;
import gr.cite.tools.logging.MapLogEntry;
import gr.cite.tools.validation.ValidatorFactory;
import jakarta.persistence.EntityManager;
import org.apache.commons.io.FilenameUtils;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.springframework.util.unit.DataSize;
import org.springframework.web.multipart.MultipartFile;

import javax.management.InvalidApplicationException;
import java.io.IOException;
import java.net.URLConnection;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;

@Service
public class DescriptionServiceImpl implements DescriptionService {

    private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DescriptionServiceImpl.class));
    private final EntityManager entityManager;
    private final AuthorizationService authorizationService;
    private final DeleterFactory deleterFactory;
    private final BuilderFactory builderFactory;
    private final ConventionService conventionService;
    private final ErrorThesaurusProperties errors;
    private final MessageSource messageSource;
    private final EventBroker eventBroker;
    private final QueryFactory queryFactory;
    private final JsonHandlingService jsonHandlingService;
    private final UserScope userScope;
    private final XmlHandlingService xmlHandlingService;
    private final FileTransformerService fileTransformerService;
    private final NotifyIntegrationEventHandler eventHandler;
    private final NotificationProperties notificationProperties;
    private final ElasticService elasticService;
    private final ValidatorFactory validatorFactory;
    private final StorageFileProperties storageFileConfig;
    private final StorageFileService storageFileService;
    private final DescriptionTouchedIntegrationEventHandler descriptionTouchedIntegrationEventHandler;
    private final AuthorizationContentResolver authorizationContentResolver;
    private final AnnotationEntityTouchedIntegrationEventHandler annotationEntityTouchedIntegrationEventHandler;
    private final AnnotationEntityRemovalIntegrationEventHandler annotationEntityRemovalIntegrationEventHandler;

    @Autowired
    public DescriptionServiceImpl(
		    EntityManager entityManager,
		    AuthorizationService authorizationService,
		    DeleterFactory deleterFactory,
		    BuilderFactory builderFactory,
		    ConventionService conventionService,
		    ErrorThesaurusProperties errors,
		    MessageSource messageSource,
		    EventBroker eventBroker,
		    QueryFactory queryFactory,
		    JsonHandlingService jsonHandlingService,
		    UserScope userScope,
		    XmlHandlingService xmlHandlingService, NotifyIntegrationEventHandler eventHandler, NotificationProperties notificationProperties, FileTransformerService fileTransformerService, ElasticService elasticService, ValidatorFactory validatorFactory, StorageFileProperties storageFileConfig, StorageFileService storageFileService, DescriptionTouchedIntegrationEventHandler descriptionTouchedIntegrationEventHandler, AuthorizationContentResolver authorizationContentResolver, AnnotationEntityTouchedIntegrationEventHandler annotationEntityTouchedIntegrationEventHandler, AnnotationEntityRemovalIntegrationEventHandler annotationEntityRemovalIntegrationEventHandler) {
        this.entityManager = entityManager;
        this.authorizationService = authorizationService;
        this.deleterFactory = deleterFactory;
        this.builderFactory = builderFactory;
        this.conventionService = conventionService;
        this.errors = errors;
        this.messageSource = messageSource;
        this.eventBroker = eventBroker;
        this.queryFactory = queryFactory;
        this.jsonHandlingService = jsonHandlingService;
        this.userScope = userScope;
        this.xmlHandlingService = xmlHandlingService;
        this.eventHandler = eventHandler;
        this.notificationProperties = notificationProperties;
        this.fileTransformerService = fileTransformerService;
	    this.elasticService = elasticService;
	    this.validatorFactory = validatorFactory;
	    this.storageFileConfig = storageFileConfig;
	    this.storageFileService = storageFileService;
        this.descriptionTouchedIntegrationEventHandler = descriptionTouchedIntegrationEventHandler;
	    this.authorizationContentResolver = authorizationContentResolver;
	    this.annotationEntityTouchedIntegrationEventHandler = annotationEntityTouchedIntegrationEventHandler;
	    this.annotationEntityRemovalIntegrationEventHandler = annotationEntityRemovalIntegrationEventHandler;
    }

    @Override
    public Map<UUID, List<String>> getDescriptionSectionPermissions(DescriptionSectionPermissionResolver model)  {
        logger.debug(new MapLogEntry("get description section permissions").And("model", model));

        Map<UUID, List<String>> response = new HashMap<>();
        
        Map<UUID, AffiliatedResource> affiliatedResourceMap = this.authorizationContentResolver.descriptionsAffiliationBySections(model.getDmpId(), model.getSectionIds());
        for (UUID sectionId : model.getSectionIds().stream().distinct().toList()){
            AffiliatedResource affiliatedResource = affiliatedResourceMap.getOrDefault(sectionId, null);
            response.put(sectionId, new ArrayList<>());
            for (String permission : model.getPermissions()){
                if (this.authorizationService.authorizeAtLeastOne(List.of(affiliatedResource), permission)) response.get(sectionId).add(permission);
            }
        }
        
        return response;
    }

    //region Persist
    
    @Override
    public Description persist(DescriptionPersist model, FieldSet fields) throws MyForbiddenException, MyValidationException, MyApplicationException, MyNotFoundException, InvalidApplicationException, IOException {
        logger.debug(new MapLogEntry("persisting data description").And("model", model).And("fields", fields));

        Boolean isUpdate = this.conventionService.isValidGuid(model.getId());

        DmpDescriptionTemplateEntity dmpDescriptionTemplate = this.entityManager.find(DmpDescriptionTemplateEntity.class, model.getDmpDescriptionTemplateId());
        if (dmpDescriptionTemplate == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getDmpDescriptionTemplateId(), DmpDescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        if (isUpdate) this.authorizationService.authorizeAtLeastOneForce(List.of(this.authorizationContentResolver.descriptionAffiliation(model.getId())), Permission.EditDescription);
        else this.authorizationService.authorizeAtLeastOneForce(List.of(this.authorizationContentResolver.descriptionsAffiliationBySection(model.getDmpId(), dmpDescriptionTemplate.getSectionId())), Permission.EditDescription);

        DescriptionEntity data;
        if (isUpdate) {
            data = this.entityManager.find(DescriptionEntity.class, model.getId());
            if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
            if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(model.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
            if (data.getStatus().equals(DescriptionStatus.Finalized)) throw new MyValidationException(this.errors.getDescriptionIsFinalized().getCode(), this.errors.getDescriptionIsFinalized().getMessage());
            if (!data.getDmpId().equals(model.getDmpId())) throw new MyValidationException(this.errors.getDmpCanNotChange().getCode(), this.errors.getDmpCanNotChange().getMessage());
            if (!data.getDmpDescriptionTemplateId().equals(model.getDmpDescriptionTemplateId())) throw new MyValidationException(this.errors.getDmpDescriptionTemplateCanNotChange().getCode(), this.errors.getDmpDescriptionTemplateCanNotChange().getMessage());
        } else {
            data = new DescriptionEntity();
            data.setId(UUID.randomUUID());
            data.setIsActive(IsActive.Active);
            data.setCreatedAt(Instant.now());
            data.setCreatedById(this.userScope.getUserId());
            data.setDmpId(model.getDmpId());
            data.setDmpDescriptionTemplateId(model.getDmpDescriptionTemplateId());
        }

        DescriptionTemplateEntity descriptionTemplateEntity = this.entityManager.find(DescriptionTemplateEntity.class, model.getDescriptionTemplateId());
        if (descriptionTemplateEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getDescriptionTemplateId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        if (!dmpDescriptionTemplate.getDescriptionTemplateGroupId().equals(descriptionTemplateEntity.getGroupId())) throw new MyValidationException(this.errors.getInvalidDescriptionTemplate().getCode(), this.errors.getInvalidDescriptionTemplate().getMessage());

        DmpEntity dmp = this.entityManager.find(DmpEntity.class, data.getDmpId());
        if (dmp == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{data.getDmpId(), Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        if (dmp.getStatus().equals(DmpStatus.Finalized) && isUpdate) throw new MyValidationException(this.errors.getDmpIsFinalized().getCode(), this.errors.getDmpIsFinalized().getMessage());

        data.setLabel(model.getLabel());
        data.setStatus(model.getStatus());
        if (model.getStatus() == DescriptionStatus.Finalized) data.setFinalizedAt(Instant.now());
        data.setDescription(model.getDescription());
        data.setDescriptionTemplateId(model.getDescriptionTemplateId());
        data.setUpdatedAt(Instant.now());
        if (isUpdate) this.entityManager.merge(data);
        else this.entityManager.persist(data);

        this.entityManager.flush();

        eu.eudat.commons.types.descriptiontemplate.DefinitionEntity definition =  this.xmlHandlingService.fromXmlSafe(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition());
        Map<String, List<UUID>> fieldToReferenceMap = this.patchAndSaveReferences(this.buildDescriptionReferencePersists(model.getProperties()), data.getId(), definition);

        this.entityManager.flush();
        
        data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(model.getProperties(), definition, fieldToReferenceMap)));

        this.entityManager.merge(data);

        this.entityManager.flush();
        
        this.persistTags(data.getId(), model.getTags());
        
        if (isUpdate){
            this.sendNotification(data);
        }

        //this.deleteOldFilesAndAddNew(datasetWizardModel, userInfo); //TODO
        this.eventBroker.emit(new DescriptionTouchedEvent(data.getId()));

        this.descriptionTouchedIntegrationEventHandler.handle(DescriptionTouchedIntegrationEventHandler.buildEventFromPersistModel(model));

        this.annotationEntityTouchedIntegrationEventHandler.handleDescription(data.getId());
        
        this.elasticService.persistDescription(data);
        return this.builderFactory.builder(DescriptionBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermission).build(BaseFieldSet.build(fields, Description._id), data);
    }

    private void sendNotification(DescriptionEntity description) throws InvalidApplicationException {
        List<DmpUserEntity> existingUsers = this.queryFactory.query(DmpUserQuery.class)
                .dmpIds(description.getDmpId())
                .isActives(IsActive.Active)
                .collect();

        if (existingUsers == null || existingUsers.size() <= 1){
            return;
        }
        for (DmpUserEntity dmpUser : existingUsers) {
            if (!dmpUser.getUserId().equals(this.userScope.getUserIdSafe())){
                UserEntity user = this.queryFactory.query(UserQuery.class).ids(dmpUser.getUserId()).first();
                if (user != null){
                    this.createDescriptionNotificationEvent(description, user, NotificationContactType.EMAIL);
                    this.createDescriptionNotificationEvent(description, user, NotificationContactType.IN_APP);
                }
            }
        }
    }

    private void createDescriptionNotificationEvent(DescriptionEntity description, UserEntity user, NotificationContactType type) throws InvalidApplicationException {
        NotifyIntegrationEvent event = new NotifyIntegrationEvent();
        event.setUserId(this.userScope.getUserId());

        UserContactInfoQuery query = this.queryFactory.query(UserContactInfoQuery.class).userIds(user.getId());
        query.setOrder(new Ordering().addAscending(UserContactInfo._ordinal));

        List<ContactPair> contactPairs = new ArrayList<>();
        contactPairs.add(new ContactPair(ContactInfoType.Email, query.first().getValue()));
        NotificationContactData contactData = new NotificationContactData(contactPairs, null, null);
        event.setContactHint(jsonHandlingService.toJsonSafe(contactData));
        event.setContactTypeHint(type);

        event = this.applyNotificationType(description.getStatus(), event);
        NotificationFieldData data = new NotificationFieldData();
        List<FieldInfo> fieldInfoList = new ArrayList<>();
        fieldInfoList.add(new FieldInfo("{recipient}", DataType.String, user.getName()));
        fieldInfoList.add(new FieldInfo("{reasonName}", DataType.String, this.queryFactory.query(UserQuery.class).ids(this.userScope.getUserId()).first().getName()));
        fieldInfoList.add(new FieldInfo("{name}", DataType.String, description.getLabel()));
        fieldInfoList.add(new FieldInfo("{id}", DataType.String, description.getId().toString()));
        data.setFields(fieldInfoList);
        event.setData(jsonHandlingService.toJsonSafe(data));

        eventHandler.handle(event);
    }

    private NotifyIntegrationEvent applyNotificationType(DescriptionStatus status, NotifyIntegrationEvent event) {
        switch (status) {
            case Draft:
                event.setNotificationType(notificationProperties.getDescriptionModifiedType());
                break;
            case Finalized:
                event.setNotificationType(notificationProperties.getDescriptionFinalisedType());
                break;
            default:
                throw new MyApplicationException("Unsupported Description Status.");
        }
        return event;
    }

    @Override
    public Description persistStatus(DescriptionStatusPersist model, FieldSet fields) throws IOException {
        logger.debug(new MapLogEntry("persisting data dmp").And("model", model).And("fields", fields));

        this.authorizationService.authorizeAtLeastOneForce(List.of(this.authorizationContentResolver.descriptionAffiliation(model.getId())), Permission.EditDescription);

        DescriptionEntity data = this.entityManager.find(DescriptionEntity.class, model.getId());
        if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
        if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(model.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
        if (!data.getStatus().equals(model.getStatus())){
            if (data.getStatus().equals(DescriptionStatus.Finalized)){
                this.authorizationService.authorizeAtLeastOneForce(List.of(this.authorizationContentResolver.descriptionAffiliation(model.getId())), Permission.FinalizeDescription);
                DmpEntity dmpEntity = this.entityManager.find(DmpEntity.class, data.getDmpId());
                if (dmpEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{data.getDmpId(), DmpEntity.class.getSimpleName()}, LocaleContextHolder.getLocale()));
                if(!dmpEntity.getStatus().equals(DmpStatus.Draft)) throw new MyValidationException(this.errors.getDmpIsFinalized().getCode(), this.errors.getDmpIsFinalized().getMessage());
            }

            data.setStatus(model.getStatus());
            if (model.getStatus() == DescriptionStatus.Finalized) data.setFinalizedAt(Instant.now());
            data.setUpdatedAt(Instant.now());
            this.entityManager.merge(data);

            this.entityManager.flush();

            this.elasticService.persistDescription(data);
            this.eventBroker.emit(new DescriptionTouchedEvent(data.getId()));

            this.annotationEntityTouchedIntegrationEventHandler.handleDescription(data.getId());
        }
        return this.builderFactory.builder(DescriptionBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermission).build(BaseFieldSet.build(fields, Description._id), data);
    }

    public List<DescriptionValidationResult> validate(List<UUID> descriptionIds){
        List<DescriptionValidationResult> descriptionValidationResults = new ArrayList<>();

        List<DescriptionEntity> descriptions = this.queryFactory.query(DescriptionQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermission).ids(descriptionIds).isActive(IsActive.Active).collect();
        if (descriptions == null){
            return null;
        }

        for (DescriptionEntity description: descriptions) {
            DescriptionValidationResult descriptionValidationResult = new DescriptionValidationResult(description.getId(), DescriptionValidationOutput.Invalid);

            DescriptionPersist.DescriptionPersistValidator validator = this.validatorFactory.validator(DescriptionPersist.DescriptionPersistValidator.class);
            validator.validate(this.buildDescriptionPersist(description));
            if (validator.result().isValid()) descriptionValidationResult.setResult(DescriptionValidationOutput.Valid);;
            descriptionValidationResults.add(descriptionValidationResult);

        }
        return descriptionValidationResults;
    }

    private @NotNull PropertyDefinitionEntity buildPropertyDefinitionEntity(PropertyDefinitionPersist persist, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity definition,  Map<String, List<UUID>> fieldToReferenceMap){
        PropertyDefinitionEntity data = new PropertyDefinitionEntity();
        if (persist == null) return data;
        if (persist.getFieldSets() != null && !persist.getFieldSets().isEmpty()){
            data.setFieldSets(new HashMap<>());
            for (String key: persist.getFieldSets().keySet()) {
                FieldSetEntity fieldSetEntity = definition != null ? definition.getFieldSetById(key).stream().findFirst().orElse(null) : null;
                data.getFieldSets().put(key, this.buildPropertyDefinitionFieldSetEntity(persist.getFieldSets().get(key), fieldSetEntity, fieldToReferenceMap));
            }
        }
        return data;
    }

    private @NotNull PropertyDefinitionFieldSetEntity buildPropertyDefinitionFieldSetEntity(PropertyDefinitionFieldSetPersist persist, FieldSetEntity fieldSetEntity,  Map<String, List<UUID>> fieldToReferenceMap){
        PropertyDefinitionFieldSetEntity data = new PropertyDefinitionFieldSetEntity();
        if (persist == null) return data;
        if (!this.conventionService.isListNullOrEmpty(persist.getItems())){
            data.setItems(new ArrayList<>());
            for (PropertyDefinitionFieldSetItemPersist itemsPersist: persist.getItems()) {
                data.getItems().add(this.buildPropertyDefinitionFieldSetItemEntity(itemsPersist, fieldSetEntity, fieldToReferenceMap));
            }
        }
        return data;
    }

    private @NotNull PropertyDefinitionFieldSetItemEntity buildPropertyDefinitionFieldSetItemEntity(PropertyDefinitionFieldSetItemPersist persist, FieldSetEntity fieldSetEntity,  Map<String, List<UUID>> fieldToReferenceMap){
        PropertyDefinitionFieldSetItemEntity data = new PropertyDefinitionFieldSetItemEntity();
        if (persist == null) return data;
        if (persist.getFields() != null && !persist.getFields().isEmpty()){
            data.setOrdinal(persist.getOrdinal());
            data.setComment(persist.getComment());
            data.setFields(new HashMap<>());
            for (String key: persist.getFields().keySet()) {
                eu.eudat.commons.types.descriptiontemplate.FieldEntity fieldEntity = fieldSetEntity != null ? fieldSetEntity.getAllField().stream().findFirst().orElse(null) : null;
                data.getFields().put(key, this.buildFieldEntity(persist.getFields().get(key), fieldEntity, fieldToReferenceMap));
            }
        }
        return data;
    }
    
    private @NotNull FieldEntity buildFieldEntity(FieldPersist persist, eu.eudat.commons.types.descriptiontemplate.FieldEntity fieldEntity,  Map<String, List<UUID>> fieldToReferenceMap){
        FieldType fieldType = fieldEntity != null && fieldEntity.getData() != null ? fieldEntity.getData().getFieldType() :  FieldType.FREE_TEXT;
        FieldEntity data = new FieldEntity();
        if (persist == null) return data;

        if (FieldType.isTextType(fieldType)) {
            if (FieldType.UPLOAD.equals(fieldType) && !this.conventionService.isNullOrEmpty(persist.getTextValue())){
                StorageFile storageFile = this.storageFileService.copyToStorage(UUID.fromString(persist.getTextValue()), StorageType.Main, true, new BaseFieldSet().ensure(StorageFile._id)); 
                this.storageFileService.updatePurgeAt(storageFile.getId(), null);
                data.setTextValue(storageFile.getId().toString());
            } else {
                data.setTextValue(persist.getTextValue());
            }
        }
        else if (FieldType.isTextListType(fieldType)) {
            if (FieldType.INTERNAL_ENTRIES_DMPS.equals(fieldType) && !this.conventionService.isListNullOrEmpty(persist.getTextListValue())){
                List<UUID> ids = persist.getTextListValue().stream().map(UUID::fromString).toList();
                Set<UUID> existingIds = this.queryFactory.query(DmpQuery.class).ids(ids).isActive(IsActive.Active).collectAs(new BaseFieldSet().ensure(Dmp._id)).stream().map(DmpEntity::getId).collect(Collectors.toSet());
                for (UUID id : ids){
                    if (!existingIds.contains(id)) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));
                }
            } if (FieldType.INTERNAL_ENTRIES_DESCRIPTIONS.equals(fieldType) && !this.conventionService.isListNullOrEmpty(persist.getTextListValue())){
                List<UUID> ids = persist.getTextListValue().stream().map(UUID::fromString).toList();
                Set<UUID> existingIds = this.queryFactory.query(DescriptionQuery.class).ids(ids).isActive(IsActive.Active).collectAs(new BaseFieldSet().ensure(Description._id)).stream().map(DescriptionEntity::getId).collect(Collectors.toSet());
                for (UUID id : ids){
                    if (!existingIds.contains(id)) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
                }
            }
            data.setTextListValue(persist.getTextListValue());
        }
        else if (FieldType.isReferenceType(fieldType) && fieldEntity != null ) {
            List<UUID> referenceIds = fieldToReferenceMap.getOrDefault(fieldEntity.getId(), null);
            if (referenceIds != null) data.setTextListValue(referenceIds.stream().map(UUID::toString).toList());
        }
        else if (FieldType.isDateType(fieldType))  data.setDateValue(persist.getDateValue());
        else if (FieldType.isExternalIdentifierType(fieldType) && persist.getExternalIdentifier() != null)  data.setExternalIdentifier(this.buildExternalIdentifierEntity(persist.getExternalIdentifier()));
       
        return data;
    }

    private @NotNull ExternalIdentifierEntity buildExternalIdentifierEntity(ExternalIdentifierPersist persist){
        ExternalIdentifierEntity data = new ExternalIdentifierEntity();
        if (persist == null) return data;

        data.setIdentifier(persist.getIdentifier());
        data.setType(persist.getType());
        return data;
    }

    private @NotNull  List<DescriptionReferencePersist> buildDescriptionReferencePersists(PropertyDefinitionPersist persist){
        List<DescriptionReferencePersist> descriptionReferencePersists = new ArrayList<>();
        if (persist.getFieldSets() != null && !persist.getFieldSets().isEmpty()){
            for (PropertyDefinitionFieldSetPersist propertyDefinitionFieldSetPersist: persist.getFieldSets().values()) {
                if (!this.conventionService.isListNullOrEmpty( propertyDefinitionFieldSetPersist.getItems())) {
                    for (PropertyDefinitionFieldSetItemPersist definitionFieldSetItemPersist : propertyDefinitionFieldSetPersist.getItems()) {
                        if (definitionFieldSetItemPersist.getFields() != null && !definitionFieldSetItemPersist.getFields().isEmpty()) {
                            for (String key : definitionFieldSetItemPersist.getFields().keySet()) {
                                FieldPersist fieldPersist = definitionFieldSetItemPersist.getFields().get(key);
                                BuildDescriptionReferencePersist(key, fieldPersist, descriptionReferencePersists);
                            }
                        }
                    }
                }
            }
        }
        return descriptionReferencePersists;
    }

    private void BuildDescriptionReferencePersist(String fieldId, FieldPersist fieldPersist, List<DescriptionReferencePersist> descriptionReferencePersists) {
        if (fieldPersist.getReference() != null) {
            if (fieldPersist.getReferences() == null) fieldPersist.setReferences(new ArrayList<>());
            fieldPersist.getReferences().add(fieldPersist.getReference());
        }
        if (!this.conventionService.isListNullOrEmpty(fieldPersist.getReferences())) {
            for (ReferencePersist referencePersist : fieldPersist.getReferences()) {
                DescriptionReferencePersist descriptionReferencePersist = new DescriptionReferencePersist();
                descriptionReferencePersist.setData(new DescriptionReferenceDataPersist());
                descriptionReferencePersist.getData().setFieldId(fieldId);
                descriptionReferencePersist.setReference(referencePersist);
                descriptionReferencePersists.add(descriptionReferencePersist);
            }
        }
    }

    private Map<String, List<UUID>> patchAndSaveReferences(List<DescriptionReferencePersist> models, UUID descriptionId, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity definition) throws InvalidApplicationException {
        if (models == null) models = new ArrayList<>();
        
        Map<String, List<UUID>> fieldToReferenceMap = new HashMap<>();

        List<DescriptionReferenceEntity> descriptionReferences = this.queryFactory.query(DescriptionReferenceQuery.class).descriptionIds(descriptionId).collect();
        Map<UUID, List<DescriptionReferenceEntity>> descriptionReferenceEntityByReferenceId = new HashMap<>();
        for (DescriptionReferenceEntity descriptionReferenceEntity : descriptionReferences){
            List<DescriptionReferenceEntity> descriptionReferenceEntities = descriptionReferenceEntityByReferenceId.getOrDefault(descriptionReferenceEntity.getReferenceId(), null);
            if (descriptionReferenceEntities == null) {
                descriptionReferenceEntities = new ArrayList<>();
                descriptionReferenceEntityByReferenceId.put(descriptionReferenceEntity.getReferenceId(), descriptionReferenceEntities);
            }
            descriptionReferenceEntities.add(descriptionReferenceEntity);
        }

        Map<UUID, DescriptionReferenceDataEntity> descriptionReferenceDataEntityMap = new HashMap<>();
        for (DescriptionReferenceEntity descriptionReferenceEntity : descriptionReferences){
            descriptionReferenceDataEntityMap.put(descriptionReferenceEntity.getId(), this.jsonHandlingService.fromJsonSafe(DescriptionReferenceDataEntity.class, descriptionReferenceEntity.getData()));
        }

        List<UUID> updatedCreatedIds = new ArrayList<>();
        for (DescriptionReferencePersist model : models) {
            ReferencePersist referencePersist = model.getReference();
            ReferenceEntity referenceEntity = null;
            if (this.conventionService.isValidGuid(referencePersist.getId())){
                referenceEntity = this.entityManager.find(ReferenceEntity.class, referencePersist.getId());
                if (referenceEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{referencePersist.getId(), Reference.class.getSimpleName()}, LocaleContextHolder.getLocale()));
            } else {
                ReferenceTypeDataEntity fieldEntity = definition.getFieldById(model.getData().getFieldId()).stream().filter(x-> x.getData() != null && x.getData().getFieldType().equals(FieldType.REFERENCE_TYPES)).map(x-> (ReferenceTypeDataEntity)x.getData()).findFirst().orElse(null);
                if (fieldEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getData().getFieldId(), ReferenceTypeDataEntity.class.getSimpleName()}, LocaleContextHolder.getLocale()));

                referenceEntity = this.queryFactory.query(ReferenceQuery.class).sourceTypes(referencePersist.getSourceType()).typeIds(fieldEntity.getReferenceTypeId()).sources(referencePersist.getSource()).isActive(IsActive.Active).references(referencePersist.getReference()).first();
                if (referenceEntity == null){
                    referenceEntity = new ReferenceEntity();
                    referenceEntity.setId(UUID.randomUUID());
                    referenceEntity.setIsActive(IsActive.Active);
                    referenceEntity.setCreatedAt(Instant.now());
                    referenceEntity.setTypeId(fieldEntity.getReferenceTypeId());

                    referenceEntity.setDefinition(this.xmlHandlingService.toXmlSafe(this.buildDefinitionEntity(referencePersist.getDefinition())));
                    referenceEntity.setUpdatedAt(Instant.now());
                    referenceEntity.setReference(referencePersist.getReference());
                    referenceEntity.setAbbreviation(referencePersist.getAbbreviation());
                    referenceEntity.setSource(referencePersist.getSource());
                    referenceEntity.setSourceType(referencePersist.getSourceType());
                    this.entityManager.persist(referenceEntity);
                }
            }

            DescriptionReferenceEntity data = null;
            List<DescriptionReferenceEntity> descriptionReferenceEntities = descriptionReferenceEntityByReferenceId.getOrDefault(referenceEntity.getId(), new ArrayList<>());
            for (DescriptionReferenceEntity descriptionReferenceEntity : descriptionReferenceEntities){
                DescriptionReferenceDataEntity descriptionReferenceDataEntity = descriptionReferenceDataEntityMap.getOrDefault(descriptionReferenceEntity.getId(), new DescriptionReferenceDataEntity());
                if (Objects.equals(descriptionReferenceDataEntity.getFieldId(), model.getData().getFieldId())){
                    data = descriptionReferenceEntity;
                    break;
                }
            }
            boolean isUpdate = data != null;

            if (!isUpdate) {
                data = new DescriptionReferenceEntity();
                data.setId(UUID.randomUUID());
                data.setReferenceId(referenceEntity.getId());
                data.setDescriptionId(descriptionId);
                data.setCreatedAt(Instant.now());
                data.setIsActive(IsActive.Active);
                data.setData(this.jsonHandlingService.toJsonSafe(this.buildDescriptionReferenceDataEntity(model.getData())));
            }
            updatedCreatedIds.add(data.getId());
            
            if (model.getData() != null){
                if (!fieldToReferenceMap.containsKey(model.getData().getFieldId())) fieldToReferenceMap.put(model.getData().getFieldId(), new ArrayList<>());
                fieldToReferenceMap.get(model.getData().getFieldId()).add(referenceEntity.getId());
            }

            data.setUpdatedAt(Instant.now());

            if (isUpdate) this.entityManager.merge(data);
            else this.entityManager.persist(data);
        }
        List<DescriptionReferenceEntity> toDelete = descriptionReferences.stream().filter(x-> updatedCreatedIds.stream().noneMatch(y-> y.equals(x.getId()))).collect(Collectors.toList());
        this.deleterFactory.deleter(DescriptionReferenceDeleter.class).delete(toDelete);
        this.entityManager.flush();
        
        return fieldToReferenceMap;
    }

    private @NotNull DescriptionReferenceDataEntity buildDescriptionReferenceDataEntity(DescriptionReferenceDataPersist persist){
        DescriptionReferenceDataEntity data = new DescriptionReferenceDataEntity();
        if (persist == null) return data;
        data.setFieldId(persist.getFieldId());
        return data;
    }

    private void persistTags(UUID id, List<String> tagLabels) throws InvalidApplicationException {
        if (tagLabels == null) tagLabels = new ArrayList<>();
        tagLabels = tagLabels.stream().filter(x-> x != null && !x.isBlank()).toList();
        
        List<DescriptionTagEntity> items = this.queryFactory.query(DescriptionTagQuery.class).isActive(IsActive.Active).descriptionIds(id).collect();
        List<TagEntity> tagsAlreadyLinked = this.queryFactory.query(TagQuery.class).isActive(IsActive.Active).ids(items.stream().map(DescriptionTagEntity::getTagId).collect(Collectors.toList())).collect();
        List<String> tagLabelsToAdd = tagLabels.stream().filter(x-> tagsAlreadyLinked.stream().noneMatch(y-> y.getLabel() != null && y.getLabel().equalsIgnoreCase(x))).toList();
        List<TagEntity> existingTags = this.queryFactory.query(TagQuery.class).isActive(IsActive.Active).tags(tagLabelsToAdd).createdByIds(this.userScope.getUserId()).collect();
        
        List<UUID> updatedCreatedIds = new ArrayList<>();
        for (String tagLabel : tagLabels) {
            TagEntity alreadyLinkedTag = tagsAlreadyLinked.stream().filter(x-> x.getLabel() != null && x.getLabel().equalsIgnoreCase(tagLabel)).findFirst().orElse(null);
            if (alreadyLinkedTag != null){
                updatedCreatedIds.addAll(items.stream().filter(x-> x.getTagId().equals(alreadyLinkedTag.getId())).map(DescriptionTagEntity::getId).toList());
            } else{
                TagEntity existingTag = existingTags.stream().filter(x-> x.getLabel() != null && x.getLabel().equalsIgnoreCase(tagLabel)).findFirst().orElse(null);
                if (existingTag == null){
                    existingTag = new TagEntity();
                    existingTag.setId(UUID.randomUUID());
                    existingTag.setLabel(tagLabel);
                    existingTag.setIsActive(IsActive.Active);
                    existingTag.setCreatedAt(Instant.now());
                    existingTag.setUpdatedAt(Instant.now());
                    this.entityManager.persist(existingTag);
                }

                DescriptionTagEntity link = new DescriptionTagEntity();
                link.setId(UUID.randomUUID());
                link.setTagId(existingTag.getId());
                link.setDescriptionId(id);
                link.setIsActive(IsActive.Active);
                link.setCreatedAt(Instant.now());
                link.setUpdatedAt(Instant.now());
                this.entityManager.persist(link);
                updatedCreatedIds.add(link.getId());
            }
        }
        List<DescriptionTagEntity> toDelete = items.stream().filter(x-> updatedCreatedIds.stream().noneMatch(y-> y.equals(x.getId()))).collect(Collectors.toList());

        this.deleterFactory.deleter(DescriptionTagDeleter.class).delete(toDelete);
    }
    
    private @NotNull DefinitionEntity buildDefinitionEntity(DefinitionPersist persist){
        DefinitionEntity data = new DefinitionEntity();
        if (persist == null) return data;
        if (!this.conventionService.isListNullOrEmpty(persist.getFields())){
            data.setFields(new ArrayList<>());
            for (eu.eudat.model.persist.referencedefinition.FieldPersist fieldPersist: persist.getFields()) {
                data.getFields().add(this.buildFieldEntity(fieldPersist));
            }
        }

        return data;
    }

    private @NotNull eu.eudat.commons.types.reference.FieldEntity buildFieldEntity(eu.eudat.model.persist.referencedefinition.FieldPersist persist){
        eu.eudat.commons.types.reference.FieldEntity data = new eu.eudat.commons.types.reference.FieldEntity();
        if (persist == null) return data;

        data.setCode(persist.getCode());
        data.setDataType(persist.getDataType());
        data.setCode(persist.getCode());

        return data;
    }

    //endregion
    
    //region delete
    
    @Override
    public void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException, IOException {
        logger.debug("deleting description: {}", id);

        this.authorizationService.authorizeAtLeastOneForce(List.of(this.authorizationContentResolver.descriptionAffiliation(id)), Permission.DeleteDescription);

        this.deleterFactory.deleter(DescriptionDeleter.class).deleteAndSaveByIds(List.of(id), false);

        this.annotationEntityRemovalIntegrationEventHandler.handleDescription(id);
    }
    
    //endregion 

    //region clone
    
    @Override
    public void clone(UUID dmpId, UUID descriptionId) throws InvalidApplicationException, IOException {
        logger.debug("cloning description: {} with description: {}", descriptionId, dmpId);

        this.authorizationService.authorizeAtLeastOneForce(List.of(this.authorizationContentResolver.descriptionAffiliation(descriptionId)), Permission.CloneDescription);

        DescriptionEntity existing = this.queryFactory.query(DescriptionQuery.class).ids(descriptionId).isActive(IsActive.Active).first();

        DescriptionEntity newDescription = new DescriptionEntity();
        newDescription.setId(UUID.randomUUID());
        newDescription.setLabel(existing.getLabel());
        newDescription.setDescription(existing.getDescription());
        newDescription.setStatus(DescriptionStatus.Draft);
        newDescription.setProperties(existing.getProperties());
        newDescription.setDmpId(dmpId);
        newDescription.setDmpDescriptionTemplateId(existing.getDmpDescriptionTemplateId());
        newDescription.setDescriptionTemplateId(existing.getDescriptionTemplateId());
        newDescription.setCreatedById(userScope.getUserId());
        newDescription.setCreatedAt(Instant.now());
        newDescription.setUpdatedAt(Instant.now());
        newDescription.setIsActive(IsActive.Active);

        this.entityManager.persist(newDescription);

        List<DescriptionReferenceEntity> descriptionReferences = this.queryFactory.query(DescriptionReferenceQuery.class)
                .descriptionIds(existing.getId())
                .isActive(IsActive.Active)
                .collect();

        List<DescriptionTagEntity> descriptionTags = this.queryFactory.query(DescriptionTagQuery.class)
                .descriptionIds(existing.getId())
                .isActive(IsActive.Active)
                .collect();

        for (DescriptionReferenceEntity descriptionReference : descriptionReferences) {
            DescriptionReferenceEntity newReference = new DescriptionReferenceEntity();
            newReference.setId(UUID.randomUUID());
            newReference.setDescriptionId(newDescription.getId());
            newReference.setReferenceId(descriptionReference.getReferenceId());
            newReference.setCreatedAt(Instant.now());
            newReference.setUpdatedAt(Instant.now());
            newReference.setIsActive(IsActive.Active);

            this.entityManager.persist(newReference);
        }

        for(DescriptionTagEntity descriptionTag : descriptionTags) {
            DescriptionTagEntity newTag = new DescriptionTagEntity();
            newTag.setId(UUID.randomUUID());
            newTag.setDescriptionId(newDescription.getId());
            newTag.setTagId(descriptionTag.getTagId());
            newTag.setCreatedAt(Instant.now());
            newTag.setUpdatedAt(Instant.now());
            newTag.setIsActive(IsActive.Active);

            this.entityManager.persist(newTag);
        }

        this.entityManager.flush();

        this.elasticService.persistDescription(newDescription);

        this.annotationEntityTouchedIntegrationEventHandler.handleDescription(newDescription.getId());
        this.annotationEntityTouchedIntegrationEventHandler.handleDescription(existing.getId());
    }
    
    //endregion

    //region file export

    @Override
    public ResponseEntity<byte[]> export(UUID id, String exportType) throws InvalidApplicationException, IOException {
        HttpHeaders headers = new HttpHeaders();

        FileEnvelope fileEnvelope = this.fileTransformerService.exportDescription(id, null, exportType); //TODO get repo from config
        headers.add("Content-Disposition", "attachment;filename=" + fileEnvelope.getFilename());
        byte[] data = fileEnvelope.getFile();
        headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);
        return new ResponseEntity<>(data, headers, HttpStatus.OK);
    }
    //endregion



    @Override
    public StorageFile uploadFieldFile(DescriptionFieldFilePersist model, MultipartFile file, FieldSet fields) throws IOException {
        //this.authorizationService.authorizeAtLeastOneForce(List.of(this.authorizationContentResolver.descriptionAffiliation(descriptionId)), Permission.CloneDescription);
        this.authorizationService.authorizeForce(Permission.EditDescription);//TODO: Missing Description or dmp for authz
        
        DescriptionTemplateEntity descriptionTemplate = this.queryFactory.query(DescriptionTemplateQuery.class).ids(model.getDescriptionTemplateId()).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermission).first();
        if (descriptionTemplate == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getDescriptionTemplateId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        eu.eudat.commons.types.descriptiontemplate.DefinitionEntity definition =  this.xmlHandlingService.fromXmlSafe(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplate.getDefinition());
        if (definition == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getDescriptionTemplateId(), eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        eu.eudat.commons.types.descriptiontemplate.FieldEntity fieldEntity = definition.getFieldById(model.getFieldId()).stream().filter(x -> x != null && x.getData() != null && x.getData().getFieldType().equals(FieldType.UPLOAD)).findFirst().orElse(null);
        if (fieldEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getFieldId(), eu.eudat.commons.types.descriptiontemplate.FieldEntity.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        UploadDataEntity uploadDataEntity = (UploadDataEntity)fieldEntity.getData();
        if (DataSize.ofBytes(file.getSize()).equals(DataSize.ofMegabytes(uploadDataEntity.getMaxFileSizeInMB()))) {
            throw new MyValidationException("The uploaded file is too large");
        }
        if(!this.conventionService.isListNullOrEmpty(uploadDataEntity.getTypes())) {
            boolean isContentTypeAccepted = false;
            for (UploadDataEntity.UploadDataOptionEntity option: uploadDataEntity.getTypes()) {
                if(Objects.equals(file.getContentType(), option.getValue())) {
                    isContentTypeAccepted = true;
                    break;
                }
            }
            if (!isContentTypeAccepted){
                throw new MyValidationException("The uploaded file has an unaccepted type");
            }
        }
        StorageFilePersist storageFilePersist = new StorageFilePersist();
        storageFilePersist.setName(FilenameUtils.removeExtension(file.getName()));
        storageFilePersist.setExtension(FilenameUtils.getExtension(file.getName()));
        storageFilePersist.setMimeType(URLConnection.guessContentTypeFromName(file.getName()));
        storageFilePersist.setOwnerId(this.userScope.getUserIdSafe());
        storageFilePersist.setStorageType(StorageType.Temp);
        storageFilePersist.setLifetime(Duration.ofSeconds(this.storageFileConfig.getTempStoreLifetimeSeconds()));
        this.validatorFactory.validator(StorageFilePersist.StorageFilePersistValidator.class).validateForce(storageFilePersist);
        return this.storageFileService.persistBytes(storageFilePersist, file.getBytes(), BaseFieldSet.build(fields, StorageFile._id, StorageFile._name));
    }

    @Override
    public StorageFileEntity getFieldFile(UUID descriptionId, UUID storageFileId) {
        this.authorizationService.authorizeAtLeastOneForce(List.of(this.authorizationContentResolver.descriptionAffiliation(descriptionId)), Permission.BrowseDescription);
        
        DescriptionEntity descriptionEntity = this.queryFactory.query(DescriptionQuery.class).isActive(IsActive.Active).ids(descriptionId).first();
        if (descriptionEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{descriptionId, Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
        
        DmpDescriptionTemplateEntity dmpDescriptionTemplateEntity = this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntity.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first();
        if (dmpDescriptionTemplateEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{descriptionEntity.getDmpDescriptionTemplateId(), DmpDescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));
        
        DmpEntity dmpEntity = this.queryFactory.query(DmpQuery.class).ids(dmpDescriptionTemplateEntity.getDmpId()).isActive(IsActive.Active).first();
        if (dmpEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{dmpDescriptionTemplateEntity.getDmpId(), Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        if (!dmpEntity.getAccessType().equals(DmpAccessType.Public))
        {
            boolean isDmpUser = this.queryFactory.query(DmpUserQuery.class).dmpIds(dmpEntity.getId()).userIds(this.userScope.getUserIdSafe()).isActives(IsActive.Active).count() > 0;
            if (!isDmpUser) throw new MyUnauthorizedException();
        }
        
        StorageFileEntity storageFile = this.queryFactory.query(StorageFileQuery.class).ids(storageFileId).first();
        if (storageFile == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{storageFileId, StorageFile.class.getSimpleName()}, LocaleContextHolder.getLocale()));
        
        return storageFile;
    }

    //region build persist

    private @NotNull DescriptionPersist buildDescriptionPersist(DescriptionEntity data){
        DescriptionPersist persist = new DescriptionPersist();
        if (data == null) return persist;

        DescriptionTemplateEntity descriptionTemplateEntity = this.entityManager.find(DescriptionTemplateEntity.class, data.getDescriptionTemplateId());
        if (descriptionTemplateEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{data.getDescriptionTemplateId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        persist.setLabel(data.getLabel());
        persist.setStatus(data.getStatus());
        persist.setDescription(data.getDescription());
        persist.setDescriptionTemplateId(data.getDescriptionTemplateId());
        persist.setDmpId(data.getDmpId());
        persist.setDmpDescriptionTemplateId(data.getDmpDescriptionTemplateId());

        eu.eudat.commons.types.descriptiontemplate.DefinitionEntity definition =  this.xmlHandlingService.fromXmlSafe(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition());

        persist.setProperties(this.buildPropertyDefinitionPersist( this.jsonHandlingService.fromJsonSafe(PropertyDefinitionEntity.class, data.getProperties()), definition));

        return persist;
    }

    private @NotNull PropertyDefinitionPersist buildPropertyDefinitionPersist(PropertyDefinitionEntity data, eu.eudat.commons.types.descriptiontemplate.DefinitionEntity definition){
        PropertyDefinitionPersist persist = new PropertyDefinitionPersist();
        if (data == null) return persist;
        if (data.getFieldSets() != null && !data.getFieldSets().isEmpty()){
            persist.setFieldSets(new HashMap<>());
            for (String key: data.getFieldSets().keySet()) {
                FieldSetEntity fieldSetEntity = definition != null ? definition.getFieldSetById(key).stream().findFirst().orElse(null) : null;
                persist.getFieldSets().put(key, this.buildPropertyDefinitionFieldSetPersist(data.getFieldSets().get(key), fieldSetEntity));
            }
        }
        return persist;
    }

    private @NotNull PropertyDefinitionFieldSetPersist buildPropertyDefinitionFieldSetPersist(PropertyDefinitionFieldSetEntity data, FieldSetEntity fieldSetEntity){
        PropertyDefinitionFieldSetPersist persist = new PropertyDefinitionFieldSetPersist();
        if (data == null) return persist;
        if (!this.conventionService.isListNullOrEmpty(data.getItems())){
            persist.setItems(new ArrayList<>());
            for (PropertyDefinitionFieldSetItemEntity itemsPersist: data.getItems()) {
                persist.getItems().add(this.buildPropertyDefinitionFieldSetItemPersist(itemsPersist, fieldSetEntity));
            }
        }
        return persist;
    }

    private @NotNull PropertyDefinitionFieldSetItemPersist buildPropertyDefinitionFieldSetItemPersist(PropertyDefinitionFieldSetItemEntity data, FieldSetEntity fieldSetEntity){
        PropertyDefinitionFieldSetItemPersist persist = new PropertyDefinitionFieldSetItemPersist();
        if (data == null) return persist;
        if (data.getFields() != null && !data.getFields().isEmpty()){
            persist.setOrdinal(data.getOrdinal());
            persist.setComment(data.getComment());
            persist.setFields(new HashMap<>());
            for (String key: data.getFields().keySet()) {
                eu.eudat.commons.types.descriptiontemplate.FieldEntity fieldEntity = fieldSetEntity != null ? fieldSetEntity.getAllField().stream().findFirst().orElse(null) : null;
                persist.getFields().put(key, this.buildFieldPersist(data.getFields().get(key), fieldEntity));
            }
        }
        return persist;
    }

    private @NotNull FieldPersist buildFieldPersist(FieldEntity data, eu.eudat.commons.types.descriptiontemplate.FieldEntity fieldEntity) {
        FieldType fieldType = fieldEntity != null && fieldEntity.getData() != null ? fieldEntity.getData().getFieldType() :  FieldType.FREE_TEXT;

        FieldPersist persist = new FieldPersist();
        if (data == null) return persist;

        if (FieldType.isTextType(fieldType)) persist.setTextValue(data.getTextValue());
        else if (FieldType.isTextListType(fieldType)) persist.setTextListValue(data.getTextListValue());
        else if (FieldType.isDateType(fieldType))  persist.setDateValue(persist.getDateValue());
        else if (FieldType.isExternalIdentifierType(fieldType) && data.getExternalIdentifier() != null)  persist.setExternalIdentifier(this.buildExternalIdentifierPersist(data.getExternalIdentifier()));
        else if (FieldType.isReferenceType(fieldType) && fieldEntity != null ) persist.setTextListValue(data.getTextListValue());

        return persist;
    }

    private @NotNull ExternalIdentifierPersist buildExternalIdentifierPersist(ExternalIdentifierEntity data){
        ExternalIdentifierPersist persist = new ExternalIdentifierPersist();
        if (data == null) return persist;

        persist.setIdentifier(data.getIdentifier());
        persist.setType(data.getType());
        return persist;
    }
}