argos/dmp-backend/core/src/main/java/eu/eudat/service/description/DescriptionServiceImpl.java

package eu.eudat.service.description;

import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.ContactInfoType;
import eu.eudat.commons.enums.DescriptionStatus;
import eu.eudat.commons.enums.DmpStatus;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.enums.notification.NotificationContactType;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.commons.types.description.FieldEntity;
import eu.eudat.commons.types.description.PropertyDefinitionEntity;
import eu.eudat.commons.types.description.PropertyDefinitionFieldSetEntity;
import eu.eudat.commons.types.description.PropertyDefinitionFieldSetItemEntity;
import eu.eudat.commons.types.notification.*;
import eu.eudat.commons.types.reference.DefinitionEntity;
import eu.eudat.configurations.notification.NotificationProperties;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.*;
import eu.eudat.errorcode.ErrorThesaurusProperties;
import eu.eudat.event.DescriptionTouchedEvent;
import eu.eudat.event.EventBroker;
import eu.eudat.integrationevent.outbox.notification.NotifyIntegrationEvent;
import eu.eudat.integrationevent.outbox.notification.NotifyIntegrationEventHandler;
import eu.eudat.model.*;
import eu.eudat.model.builder.DescriptionBuilder;
import eu.eudat.model.deleter.DescriptionDeleter;
import eu.eudat.model.deleter.DescriptionReferenceDeleter;
import eu.eudat.model.deleter.DescriptionTagDeleter;
import eu.eudat.model.file.FileEnvelope;
import eu.eudat.model.persist.DescriptionPersist;
import eu.eudat.model.persist.DescriptionReferencePersist;
import eu.eudat.model.persist.DescriptionStatusPersist;
import eu.eudat.model.persist.ReferencePersist;
import eu.eudat.model.persist.descriptionproperties.FieldPersist;
import eu.eudat.model.persist.descriptionproperties.PropertyDefinitionFieldSetItemPersist;
import eu.eudat.model.persist.descriptionproperties.PropertyDefinitionFieldSetPersist;
import eu.eudat.model.persist.descriptionproperties.PropertyDefinitionPersist;
import eu.eudat.model.persist.referencedefinition.DefinitionPersist;
import eu.eudat.query.*;
import eu.eudat.service.elastic.ElasticService;
import eu.eudat.service.transformer.FileTransformerService;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException;
import gr.cite.tools.exception.MyValidationException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.fieldset.FieldSet;
import gr.cite.tools.logging.LoggerService;
import gr.cite.tools.logging.MapLogEntry;
import jakarta.persistence.EntityManager;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;

import javax.management.InvalidApplicationException;
import java.io.IOException;
import java.nio.file.Files;
import java.time.Instant;
import java.util.*;
import java.util.stream.Collectors;

@Service
public class DescriptionServiceImpl implements DescriptionService {

    private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DescriptionServiceImpl.class));

    private final EntityManager entityManager;

    private final AuthorizationService authorizationService;

    private final DeleterFactory deleterFactory;

    private final BuilderFactory builderFactory;

    private final ConventionService conventionService;

    private final ErrorThesaurusProperties errors;

    private final MessageSource messageSource;

    private final EventBroker eventBroker;

    private final QueryFactory queryFactory;

    private final JsonHandlingService jsonHandlingService;

    private final UserScope userScope;
    private final XmlHandlingService xmlHandlingService;
    private final FileTransformerService fileTransformerService;

    private final NotifyIntegrationEventHandler eventHandler;

    private final NotificationProperties notificationProperties;

    private final ElasticService elasticService;


    @Autowired
    public DescriptionServiceImpl(
            EntityManager entityManager,
            AuthorizationService authorizationService,
            DeleterFactory deleterFactory,
            BuilderFactory builderFactory,
            ConventionService conventionService,
            ErrorThesaurusProperties errors,
            MessageSource messageSource,
            EventBroker eventBroker,
            QueryFactory queryFactory,
            JsonHandlingService jsonHandlingService,
            UserScope userScope,
            XmlHandlingService xmlHandlingService, NotifyIntegrationEventHandler eventHandler, NotificationProperties notificationProperties, FileTransformerService fileTransformerService, ElasticService elasticService) {
        this.entityManager = entityManager;
        this.authorizationService = authorizationService;
        this.deleterFactory = deleterFactory;
        this.builderFactory = builderFactory;
        this.conventionService = conventionService;
        this.errors = errors;
        this.messageSource = messageSource;
        this.eventBroker = eventBroker;
        this.queryFactory = queryFactory;
        this.jsonHandlingService = jsonHandlingService;
        this.userScope = userScope;
        this.xmlHandlingService = xmlHandlingService;
        this.eventHandler = eventHandler;
        this.notificationProperties = notificationProperties;
        this.fileTransformerService = fileTransformerService;
	    this.elasticService = elasticService;
    }

    //region Persist
    
    @Override
    public Description persist(DescriptionPersist model, FieldSet fields) throws MyForbiddenException, MyValidationException, MyApplicationException, MyNotFoundException, InvalidApplicationException, IOException {
        logger.debug(new MapLogEntry("persisting data dmp").And("model", model).And("fields", fields));

        this.authorizationService.authorizeForce(Permission.EditDescription);

        Boolean isUpdate = this.conventionService.isValidGuid(model.getId());

        DescriptionEntity data;
        if (isUpdate) {
            data = this.entityManager.find(DescriptionEntity.class, model.getId());
            if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
            if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(model.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
            if (data.getStatus().equals(DescriptionStatus.Finalized)) throw new MyValidationException(this.errors.getDescriptionIsFinalized().getCode(), this.errors.getDescriptionIsFinalized().getMessage());
            if (!data.getDmpId().equals(model.getDmpId())) throw new MyValidationException(this.errors.getDmpCanNotChange().getCode(), this.errors.getDmpCanNotChange().getMessage());
            if (!data.getDmpDescriptionTemplateId().equals(model.getDmpDescriptionTemplateId())) throw new MyValidationException(this.errors.getDmpDescriptionTemplateCanNotChange().getCode(), this.errors.getDmpDescriptionTemplateCanNotChange().getMessage());
        } else {
            data = new DescriptionEntity();
            data.setId(UUID.randomUUID());
            data.setIsActive(IsActive.Active);
            data.setCreatedAt(Instant.now());
            data.setDmpId(model.getDmpId());
            data.setDmpDescriptionTemplateId(model.getDmpDescriptionTemplateId());
        }

        DmpDescriptionTemplateEntity dmpDescriptionTemplate = this.entityManager.find(DmpDescriptionTemplateEntity.class, data.getDescriptionTemplateId());
        if (dmpDescriptionTemplate == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{data.getDescriptionTemplateId(), DmpDescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));
        
        DescriptionTemplateEntity descriptionTemplateEntity = this.entityManager.find(DescriptionTemplateEntity.class, model.getDescriptionTemplateId());
        if (descriptionTemplateEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getDescriptionTemplateId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        if (!dmpDescriptionTemplate.getDescriptionTemplateGroupId().equals(descriptionTemplateEntity.getGroupId())) throw new MyValidationException(this.errors.getInvalidDescriptionTemplate().getCode(), this.errors.getInvalidDescriptionTemplate().getMessage());

        DmpEntity dmp = this.entityManager.find(DmpEntity.class, data.getDmpId());
        if (dmp == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{data.getDmpId(), Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        if (dmp.getStatus().equals(DmpStatus.Finalized) && isUpdate) throw new MyValidationException(this.errors.getDmpIsFinalized().getCode(), this.errors.getDmpIsFinalized().getMessage());

        data.setLabel(model.getLabel());
        data.setProperties(this.jsonHandlingService.toJson(this.buildPropertyDefinitionEntity(model.getProperties())));
        data.setStatus(model.getStatus());
        data.setDescription(model.getDescription());
        data.setDescriptionTemplateId(model.getDescriptionTemplateId());
        data.setUpdatedAt(Instant.now());
        if (isUpdate) this.entityManager.merge(data);
        else this.entityManager.persist(data);

        this.persistTags(data.getId(), model.getTags());
        this.persistDescriptionReferences(data.getId(), model.getReferences());
        
        this.entityManager.flush();
        
        if (isUpdate){
            this.sendNotification(data);
        }

        //this.deleteOldFilesAndAddNew(datasetWizardModel, userInfo); //TODO
        this.eventBroker.emit(new DescriptionTouchedEvent(data.getId()));

        this.elasticService.persistDescription(data);
        return this.builderFactory.builder(DescriptionBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(BaseFieldSet.build(fields, Description._id), data);
    }

    private void sendNotification(DescriptionEntity description) throws InvalidApplicationException {
        List<DmpUserEntity> existingUsers = this.queryFactory.query(DmpUserQuery.class)
                .dmpIds(description.getDmpId())
                .isActives(IsActive.Active)
                .collect();

        if (existingUsers == null || existingUsers.size() <= 1){
            return;
        }
        for (DmpUserEntity dmpUser : existingUsers) {
            if (!dmpUser.getUserId().equals(this.userScope.getUserIdSafe())){
                UserEntity user = this.queryFactory.query(UserQuery.class).ids(dmpUser.getUserId()).first();
                if (user != null){
                    this.createDescriptionNotificationEvent(description, user, NotificationContactType.EMAIL);
                    this.createDescriptionNotificationEvent(description, user, NotificationContactType.IN_APP);
                }
            }
        }
    }

    private void createDescriptionNotificationEvent(DescriptionEntity description, UserEntity user, NotificationContactType type) throws InvalidApplicationException {
        NotifyIntegrationEvent event = new NotifyIntegrationEvent();
        event.setUserId(this.userScope.getUserId());

        UserContactInfoQuery query = this.queryFactory.query(UserContactInfoQuery.class).userIds(user.getId());
        query.setOrder(new Ordering().addAscending(UserContactInfo._ordinal));

        List<ContactPair> contactPairs = new ArrayList<>();
        contactPairs.add(new ContactPair(ContactInfoType.Email, query.first().getValue()));
        NotificationContactData contactData = new NotificationContactData(contactPairs, null, null);
        event.setContactHint(jsonHandlingService.toJsonSafe(contactData));
        event.setContactTypeHint(type);

        event = this.applyNotificationType(description.getStatus(), event);
        NotificationFieldData data = new NotificationFieldData();
        List<FieldInfo> fieldInfoList = new ArrayList<>();
        fieldInfoList.add(new FieldInfo("{recipient}", DataType.String, user.getName()));
        fieldInfoList.add(new FieldInfo("{reasonName}", DataType.String, this.queryFactory.query(UserQuery.class).ids(this.userScope.getUserId()).first().getName()));
        fieldInfoList.add(new FieldInfo("{name}", DataType.String, description.getLabel()));
        fieldInfoList.add(new FieldInfo("{id}", DataType.String, description.getId().toString()));
        data.setFields(fieldInfoList);
        event.setData(jsonHandlingService.toJsonSafe(data));

        eventHandler.handle(event);
    }

    private NotifyIntegrationEvent applyNotificationType(DescriptionStatus status, NotifyIntegrationEvent event) {
        switch (status) {
            case Draft:
                event.setNotificationType(notificationProperties.getDescriptionModifiedType());
            case Finalized:
                event.setNotificationType(notificationProperties.getDescriptionFinalisedType());
            default:
                throw new MyApplicationException("Unsupported Description Status.");
        }
    }

//    public List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> getFieldById(String id){
//        List<eu.eudat.commons.types.descriptiontemplate.FieldEntity> fieldEntities = new ArrayList<>();
//        if (id == null || id.isBlank()) return fieldEntities;
//        if (this.getFieldSets() != null){
//            for (FieldSetEntity fieldSetEntity: this.getFieldSets()) {
//                fieldEntities.addAll(fieldSetEntity.getFieldById(id));
//            }
//        }
//        if (this.getSections() != null){
//            for (SectionEntity sectionEntity: this.getSections()) {
//                fieldEntities.addAll(sectionEntity.getFieldById(id));
//            }
//        }
//        return fieldEntities;
//    }
    
    private void descriptionForce(DescriptionEntity description) throws Exception {
        List<String> datasetProfileValidators = new LinkedList<>();
        DescriptionTemplateEntity descriptionTemplateEntity = this.entityManager.find(DescriptionTemplateEntity.class, description.getDescriptionTemplateId());
        if (descriptionTemplateEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{description.getDescriptionTemplateId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));
        eu.eudat.commons.types.descriptiontemplate.DefinitionEntity descriptionTemplateDefinition = this.xmlHandlingService.fromXml(eu.eudat.commons.types.descriptiontemplate.DefinitionEntity.class, descriptionTemplateEntity.getDefinition());
        
        
        
        
        
        //        DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
//        DocumentBuilder builder = builderFactory.newDocumentBuilder();
//        Document xmlDocument = builder.parse(new ByteArrayInputStream(profile.getDefinition().getBytes()));
//
//        XPath xPath = XPathFactory.newInstance().newXPath();
//        String expression = "//validation/@type[.=1]/ancestor::field/@id";
//        NodeList nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);
//
//        for (int i = 0; i < nodeList.getLength(); i++) {
//            Node node = nodeList.item(i);
//            datasetProfileValidators.add(node.getNodeValue());
//        }
//
//        expression = "//validation/@type[.=1]/ancestor::fieldSet";
//        nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);
//
//
//        JSONObject obj = new JSONObject(description.getProperties());
//        VisibilityRuleService visibilityRuleService = new VisibilityRuleServiceImpl();
//        visibilityRuleService.setProperties(obj.toMap());
//
//        description.setProfile(profile);
//        PagedDatasetProfile pagedDatasetProfile = this.getPagedProfile(new DatasetWizardModel(), description);
//        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
//
//
//        String failedField = null;
//
//        for (String validator : datasetProfileValidators) {
//            if (obj.has(validator) && isNullOrEmpty(obj.getString(validator)) && isElementVisible(nodeList, validator, visibilityRuleService)) {
//                //throw new Exception("Field value of " + validator + " must be filled.");
//                failedField = validator;
//                break;
//            }
//        }

//        return failedField;
    }

    @Override
    public Description persistStatus(DescriptionStatusPersist model, FieldSet fields) throws IOException {
        logger.debug(new MapLogEntry("persisting data dmp").And("model", model).And("fields", fields));

        this.authorizationService.authorizeForce(Permission.EditDescription);

        DescriptionEntity data = this.entityManager.find(DescriptionEntity.class, model.getId());
        if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
        if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(model.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
        if (!data.getStatus().equals(model.getStatus())){
            if (data.getStatus().equals(DescriptionStatus.Finalized)){
                DmpEntity dmpEntity = this.entityManager.find(DmpEntity.class, data.getDmpId());
                if (dmpEntity == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{data.getDmpId(), DmpEntity.class.getSimpleName()}, LocaleContextHolder.getLocale()));
                if(!dmpEntity.getStatus().equals(DmpStatus.Draft)) throw new MyValidationException(this.errors.getDmpIsFinalized().getCode(), this.errors.getDmpIsFinalized().getMessage());
            }

            data.setStatus(model.getStatus());
            data.setUpdatedAt(Instant.now());
            this.entityManager.merge(data);

            this.entityManager.flush();

            this.elasticService.persistDescription(data);
            this.eventBroker.emit(new DescriptionTouchedEvent(data.getId()));
        }
        return this.builderFactory.builder(DescriptionBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermissionOrPublic).build(BaseFieldSet.build(fields, Description._id), data);
    }

    private @NotNull PropertyDefinitionEntity buildPropertyDefinitionEntity(PropertyDefinitionPersist persist){
        PropertyDefinitionEntity data = new PropertyDefinitionEntity();
        if (persist == null) return data;
        if (persist.getFieldSets() != null && !persist.getFieldSets().isEmpty()){
            data.setFieldSets(new HashMap<>());
            for (String key: persist.getFieldSets().keySet()) {
                data.getFieldSets().put(key, this.buildPropertyDefinitionFieldSetEntity(persist.getFieldSets().get(key)));
            }
        }
        return data;
    }

    private @NotNull PropertyDefinitionFieldSetEntity buildPropertyDefinitionFieldSetEntity(PropertyDefinitionFieldSetPersist persist){
        PropertyDefinitionFieldSetEntity data = new PropertyDefinitionFieldSetEntity();
        if (persist == null) return data;
        if (!this.conventionService.isListNullOrEmpty(persist.getItems())){
            data.setItems(new ArrayList<>());
            for (PropertyDefinitionFieldSetItemPersist sectionPersist: persist.getItems()) {
                data.getItems().add(this.buildPropertyDefinitionFieldSetItemEntity(sectionPersist));
            }
        }
        return data;
    }

    private @NotNull PropertyDefinitionFieldSetItemEntity buildPropertyDefinitionFieldSetItemEntity(PropertyDefinitionFieldSetItemPersist persist){
        PropertyDefinitionFieldSetItemEntity data = new PropertyDefinitionFieldSetItemEntity();
        if (persist == null) return data;
        if (persist.getFields() != null && !persist.getFields().isEmpty()){
            data.setFields(new HashMap<>());
            for (String key: persist.getFields().keySet()) {
                data.getFields().put(key, this.buildFieldEntity(persist.getFields().get(key)));
            }
        }
        return data;
    }
    
    private @NotNull FieldEntity buildFieldEntity(FieldPersist persist){
        FieldEntity data = new FieldEntity();
        if (persist == null) return data;

        data.setValue(persist.getValue());
        return data;
    }


    private void persistTags(UUID id, List<String> tagLabels) throws InvalidApplicationException {
        if (tagLabels == null) tagLabels = new ArrayList<>();
        tagLabels = tagLabels.stream().filter(x-> x != null && !x.isBlank()).toList();
        
        List<DescriptionTagEntity> items = this.queryFactory.query(DescriptionTagQuery.class).isActive(IsActive.Active).descriptionIds(id).collect();
        List<TagEntity> tagsAlreadyLinked = this.queryFactory.query(TagQuery.class).isActive(IsActive.Active).ids(items.stream().map(DescriptionTagEntity::getTagId).collect(Collectors.toList())).collect();
        List<String> tagLabelsToAdd = tagLabels.stream().filter(x-> tagsAlreadyLinked.stream().noneMatch(y-> y.getLabel() != null && y.getLabel().equalsIgnoreCase(x))).toList();
        List<TagEntity> existingTags = this.queryFactory.query(TagQuery.class).isActive(IsActive.Active).tags(tagLabelsToAdd).createdByIds(this.userScope.getUserId()).collect();
        
        List<UUID> updatedCreatedIds = new ArrayList<>();
        for (String tagLabel : tagLabels) {
            TagEntity alreadyLinkedTag = tagsAlreadyLinked.stream().filter(x-> x.getLabel() != null && x.getLabel().equalsIgnoreCase(tagLabel)).findFirst().orElse(null);
            if (alreadyLinkedTag != null){
                updatedCreatedIds.addAll(items.stream().filter(x-> x.getTagId().equals(alreadyLinkedTag.getId())).map(DescriptionTagEntity::getId).toList());
            } else{
                TagEntity existingTag = existingTags.stream().filter(x-> x.getLabel() != null && x.getLabel().equalsIgnoreCase(tagLabel)).findFirst().orElse(null);
                if (existingTag == null){
                    existingTag = new TagEntity();
                    existingTag.setId(UUID.randomUUID());
                    existingTag.setLabel(tagLabel);
                    existingTag.setIsActive(IsActive.Active);
                    existingTag.setCreatedAt(Instant.now());
                    existingTag.setUpdatedAt(Instant.now());
                    this.entityManager.persist(existingTag);
                }

                DescriptionTagEntity link = new DescriptionTagEntity();
                link.setId(UUID.randomUUID());
                link.setTagId(existingTag.getId());
                link.setDescriptionId(id);
                link.setIsActive(IsActive.Active);
                link.setCreatedAt(Instant.now());
                link.setUpdatedAt(Instant.now());
                this.entityManager.persist(link);
                updatedCreatedIds.add(link.getId());
            }
        }
        List<DescriptionTagEntity> toDelete = items.stream().filter(x-> updatedCreatedIds.stream().noneMatch(y-> y.equals(x.getId()))).collect(Collectors.toList());

        this.deleterFactory.deleter(DescriptionTagDeleter.class).delete(toDelete);
    }
    
    private void persistDescriptionReferences(UUID id, List<DescriptionReferencePersist> persists) throws InvalidApplicationException {
        if (persists == null) persists = new ArrayList<>();

        List<DescriptionReferenceEntity> existingItems = this.queryFactory.query(DescriptionReferenceQuery.class).isActive(IsActive.Active).descriptionIds(id).collect();
        Map<UUID, ReferenceEntity> existingReferencesMap = this.queryFactory.query(ReferenceQuery.class).isActive(IsActive.Active).ids(persists.stream().filter(x-> x.getReference().getId() != null).map(x-> x.getReference().getId()).toList()).collect()
            .stream()
                .collect(Collectors.toMap(ReferenceEntity::getId, x-> x));;

        List<UUID> updatedCreatedIds = new ArrayList<>();
        for (DescriptionReferencePersist persist : persists) {
            Boolean isUpdate = this.conventionService.isValidGuid(persist.getId());

            DescriptionReferenceEntity data;
            if (isUpdate) {
                data = this.entityManager.find(DescriptionReferenceEntity.class, persist.getId());
                if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{persist.getId(), DescriptionReference.class.getSimpleName()}, LocaleContextHolder.getLocale()));
                if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(persist.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
                if (!data.getReferenceId().equals(persist.getReference().getId())) throw new MyValidationException("reference can not changed");
            } else {
                data = new DescriptionReferenceEntity();
                data.setId(UUID.randomUUID());
                data.setDescriptionId(id);
                data.setCreatedAt(Instant.now());
                data.setIsActive(IsActive.Active);
            }

            ReferenceEntity reference = this.persistReference(existingReferencesMap, persist.getReference());
            data.setReferenceId(reference.getId());
            data.setUpdatedAt(Instant.now());
            
            if (isUpdate) this.entityManager.merge(data);
            else this.entityManager.persist(data);

            updatedCreatedIds.add(data.getId());
        }
        List<DescriptionReferenceEntity> toDelete = existingItems.stream().filter(x-> updatedCreatedIds.stream().noneMatch(y-> y.equals(x.getId()))).collect(Collectors.toList());

        this.deleterFactory.deleter(DescriptionReferenceDeleter.class).delete(toDelete);
    }
    
    private ReferenceEntity persistReference(Map<UUID, ReferenceEntity> existingReferencesMap , ReferencePersist persist){
        Boolean isUpdate = this.conventionService.isValidGuid(persist.getId());

        ReferenceEntity data;
        if (isUpdate) {
            data = this.entityManager.find(ReferenceEntity.class, persist.getId());
            if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{persist.getId(), Reference.class.getSimpleName()}, LocaleContextHolder.getLocale()));
            if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(persist.getHash())) throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
        } else {
            data = new ReferenceEntity();
            data.setId(UUID.randomUUID());
            data.setCreatedAt(Instant.now());
            data.setIsActive(IsActive.Active);
        }
        data.setDefinition(this.xmlHandlingService.toXmlSafe(this.buildDefinitionEntity(persist.getDefinition())));
        data.setUpdatedAt(Instant.now());
        data.setReference(persist.getReference());
        data.setAbbreviation(persist.getAbbreviation());
        data.setSource(persist.getSource());
        data.setSourceType(persist.getSourceType());
        data.setUpdatedAt(Instant.now());


        if (isUpdate) this.entityManager.merge(data);
        else this.entityManager.persist(data);
        
        return data;
    }

    private @NotNull DefinitionEntity buildDefinitionEntity(DefinitionPersist persist){
        DefinitionEntity data = new DefinitionEntity();
        if (persist == null) return data;
        if (!this.conventionService.isListNullOrEmpty(persist.getFields())){
            data.setFields(new ArrayList<>());
            for (eu.eudat.model.persist.referencedefinition.FieldPersist fieldPersist: persist.getFields()) {
                data.getFields().add(this.buildFieldEntity(fieldPersist));
            }
        }

        return data;
    }

    private @NotNull eu.eudat.commons.types.reference.FieldEntity buildFieldEntity(eu.eudat.model.persist.referencedefinition.FieldPersist persist){
        eu.eudat.commons.types.reference.FieldEntity data = new eu.eudat.commons.types.reference.FieldEntity();
        if (persist == null) return data;

        data.setCode(persist.getCode());
        data.setDataType(persist.getDataType());
        data.setCode(persist.getCode());

        return data;
    }

    //endregion
    
    //region delete
    
    @Override
    public void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException, IOException {
        logger.debug("deleting description: {}", id);

        this.authorizationService.authorizeForce(Permission.DeleteDescription);

        this.deleterFactory.deleter(DescriptionDeleter.class).deleteAndSaveByIds(List.of(id), false);
    }
    
    //endregion 

    //region clone
    
    @Override
    public void clone(UUID dmpId, UUID descriptionId) throws InvalidApplicationException, IOException {
        logger.debug("cloning description: {} with dmp: {}", descriptionId, dmpId);

        this.authorizationService.authorizeForce(Permission.CloneDescription);

        DescriptionEntity existing = this.queryFactory.query(DescriptionQuery.class).ids(descriptionId).isActive(IsActive.Active).first();

        DescriptionEntity newDescription = new DescriptionEntity();
        newDescription.setId(UUID.randomUUID());
        newDescription.setLabel(existing.getLabel());
        newDescription.setDescription(existing.getDescription());
        newDescription.setStatus(DescriptionStatus.Draft);
        newDescription.setProperties(existing.getProperties());
        newDescription.setDmpId(dmpId);
        newDescription.setDmpDescriptionTemplateId(existing.getDmpDescriptionTemplateId());
        newDescription.setDescriptionTemplateId(existing.getDescriptionTemplateId());
        newDescription.setCreatedById(userScope.getUserId());
        newDescription.setCreatedAt(Instant.now());
        newDescription.setUpdatedAt(Instant.now());
        newDescription.setIsActive(IsActive.Active);

        this.entityManager.persist(newDescription);

        List<DescriptionReferenceEntity> descriptionReferences = this.queryFactory.query(DescriptionReferenceQuery.class)
                .descriptionIds(existing.getId())
                .isActive(IsActive.Active)
                .collect();

        List<DescriptionTagEntity> descriptionTags = this.queryFactory.query(DescriptionTagQuery.class)
                .descriptionIds(existing.getId())
                .isActive(IsActive.Active)
                .collect();

        for (DescriptionReferenceEntity descriptionReference : descriptionReferences) {
            DescriptionReferenceEntity newReference = new DescriptionReferenceEntity();
            newReference.setId(UUID.randomUUID());
            newReference.setDescriptionId(newDescription.getId());
            newReference.setReferenceId(descriptionReference.getReferenceId());
            newReference.setCreatedAt(Instant.now());
            newReference.setUpdatedAt(Instant.now());
            newReference.setIsActive(IsActive.Active);

            this.entityManager.persist(newReference);
        }

        for(DescriptionTagEntity descriptionTag : descriptionTags) {
            DescriptionTagEntity newTag = new DescriptionTagEntity();
            newTag.setId(UUID.randomUUID());
            newTag.setDescriptionId(newDescription.getId());
            newTag.setTagId(descriptionTag.getTagId());
            newTag.setCreatedAt(Instant.now());
            newTag.setUpdatedAt(Instant.now());
            newTag.setIsActive(IsActive.Active);

            this.entityManager.persist(newTag);
        }

        this.entityManager.flush();

        this.elasticService.persistDescription(newDescription);

    }
    
    //endregion

    //region file export

    @Override
    public ResponseEntity<byte[]> export(UUID id, String exportType) throws InvalidApplicationException, IOException {
        HttpHeaders headers = new HttpHeaders();
        String type = exportType;

        FileEnvelope fileEnvelope = this.fileTransformerService.exportDescription(id, type);
        headers.add("Content-Disposition", "attachment;filename=" + fileEnvelope.getFilename());
        byte[] data = Files.readAllBytes(fileEnvelope.getFile().toPath());
        headers.setContentType(MediaType.APPLICATION_OCTET_STREAM);
        return new ResponseEntity<>(data, headers, HttpStatus.OK);
    }

    //endregion
}