argos/dmp-backend/core/src/main/java/eu/eudat/service/descriptiontemplate/DescriptionTemplateServiceImpl.java

package eu.eudat.service.descriptiontemplate;

import com.fasterxml.jackson.core.JsonProcessingException;
import eu.eudat.authorization.AuthorizationFlags;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.JsonHandlingService;
import eu.eudat.commons.XmlHandlingService;
import eu.eudat.commons.enums.*;
import eu.eudat.commons.enums.notification.NotificationContactType;
import eu.eudat.commons.scope.tenant.TenantScope;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.commons.types.descriptiontemplate.*;
import eu.eudat.commons.types.descriptiontemplate.fielddata.BaseFieldDataEntity;
import eu.eudat.commons.types.descriptiontemplate.fielddata.ReferenceTypeDataEntity;
import eu.eudat.commons.types.descriptiontemplate.importexport.*;
import eu.eudat.commons.types.notification.*;
import eu.eudat.configurations.notification.NotificationProperties;
import eu.eudat.convention.ConventionService;
import eu.eudat.data.*;
import eu.eudat.errorcode.ErrorThesaurusProperties;
import eu.eudat.integrationevent.outbox.notification.NotifyIntegrationEvent;
import eu.eudat.integrationevent.outbox.notification.NotifyIntegrationEventHandler;
import eu.eudat.model.*;
import eu.eudat.model.builder.DescriptionTemplateBuilder;
import eu.eudat.model.deleter.DescriptionTemplateDeleter;
import eu.eudat.model.deleter.UserDescriptionTemplateDeleter;
import eu.eudat.model.descriptiontemplatedefinition.Definition;
import eu.eudat.model.descriptiontemplatedefinition.Field;
import eu.eudat.model.descriptiontemplatedefinition.Page;
import eu.eudat.model.descriptiontemplatedefinition.Section;
import eu.eudat.model.persist.*;
import eu.eudat.model.persist.descriptiontemplatedefinition.*;
import eu.eudat.model.persist.descriptiontemplatedefinition.fielddata.BaseFieldDataPersist;
import eu.eudat.query.*;
import eu.eudat.service.fielddatahelper.FieldDataHelperService;
import eu.eudat.service.fielddatahelper.FieldDataHelperServiceProvider;
import eu.eudat.service.responseutils.ResponseUtilsService;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.builder.BuilderFactory;
import gr.cite.tools.data.deleter.DeleterFactory;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.exception.MyApplicationException;
import gr.cite.tools.exception.MyForbiddenException;
import gr.cite.tools.exception.MyNotFoundException;
import gr.cite.tools.exception.MyValidationException;
import gr.cite.tools.fieldset.BaseFieldSet;
import gr.cite.tools.fieldset.FieldSet;
import gr.cite.tools.logging.LoggerService;
import gr.cite.tools.logging.MapLogEntry;
import gr.cite.tools.validation.ValidatorFactory;
import jakarta.persistence.EntityManager;
import jakarta.xml.bind.JAXBException;
import org.jetbrains.annotations.NotNull;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.MessageSource;
import org.springframework.context.i18n.LocaleContextHolder;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import org.xml.sax.SAXException;

import javax.management.InvalidApplicationException;
import javax.ws.rs.NotSupportedException;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerException;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.UUID;
import java.util.stream.Collectors;

@Service
public class DescriptionTemplateServiceImpl implements DescriptionTemplateService {

    private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(DescriptionTemplateServiceImpl.class));

    private final EntityManager entityManager;

    private final UserScope userScope;

    private final AuthorizationService authorizationService;

    private final DeleterFactory deleterFactory;

    private final BuilderFactory builderFactory;

    private final ConventionService conventionService;

    private final MessageSource messageSource;

    private final XmlHandlingService xmlHandlingService;

    private final FieldDataHelperServiceProvider fieldDataHelperServiceProvider;

    private final QueryFactory queryFactory;

    private final ErrorThesaurusProperties errors;

    private final TenantScope tenantScope;

    private final ResponseUtilsService responseUtilsService;


    private final JsonHandlingService jsonHandlingService;

    private final NotifyIntegrationEventHandler eventHandler;

    private final NotificationProperties notificationProperties;

    private final ValidatorFactory validatorFactory;

    @Autowired
    public DescriptionTemplateServiceImpl(
            EntityManager entityManager,
            UserScope userScope, AuthorizationService authorizationService,
            DeleterFactory deleterFactory,
            BuilderFactory builderFactory,
            ConventionService conventionService,
            MessageSource messageSource,
            XmlHandlingService xmlHandlingService,
            FieldDataHelperServiceProvider fieldDataHelperServiceProvider,
            QueryFactory queryFactory, ErrorThesaurusProperties errors,
            TenantScope tenantScope,
            ResponseUtilsService responseUtilsService,
            JsonHandlingService jsonHandlingService,
            NotifyIntegrationEventHandler eventHandler,
            NotificationProperties notificationProperties,
            ValidatorFactory validatorFactory) {
        this.entityManager = entityManager;
        this.userScope = userScope;
        this.authorizationService = authorizationService;
        this.deleterFactory = deleterFactory;
        this.builderFactory = builderFactory;
        this.conventionService = conventionService;
        this.messageSource = messageSource;
        this.xmlHandlingService = xmlHandlingService;
        this.fieldDataHelperServiceProvider = fieldDataHelperServiceProvider;
        this.queryFactory = queryFactory;
        this.errors = errors;
        this.tenantScope = tenantScope;
        this.responseUtilsService = responseUtilsService;
        this.jsonHandlingService = jsonHandlingService;
        this.eventHandler = eventHandler;
        this.notificationProperties = notificationProperties;
        this.validatorFactory = validatorFactory;
    }

    //region Persist

    public DescriptionTemplate persist(DescriptionTemplatePersist model, FieldSet fields) throws MyForbiddenException, MyValidationException, MyApplicationException, MyNotFoundException, InvalidApplicationException, JAXBException, ParserConfigurationException, JsonProcessingException, TransformerException {
        logger.debug(new MapLogEntry("persisting data descriptionTemplate").And("model", model).And("fields", fields));

        this.authorizationService.authorizeForce(Permission.EditDescriptionTemplate);

        Boolean isUpdate = this.conventionService.isValidGuid(model.getId());

        DescriptionTemplateEntity data;
        if (isUpdate) {
            data = this.entityManager.find(DescriptionTemplateEntity.class, model.getId());
            if (data == null)
                throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));
            if (!this.conventionService.hashValue(data.getUpdatedAt()).equals(model.getHash()))
                throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());
            if (data.getStatus().equals(DescriptionTemplateStatus.Finalized))
                throw new MyForbiddenException("Can not update finalized template");
        } else {
            data = new DescriptionTemplateEntity();
            data.setId(UUID.randomUUID());
            data.setStatus(DescriptionTemplateStatus.Draft);
            data.setIsActive(IsActive.Active);
            data.setCreatedAt(Instant.now());
            data.setGroupId(UUID.randomUUID());
            data.setVersionStatus(DescriptionTemplateVersionStatus.NotFinalized);
            data.setVersion((short) 1);
        }

        DescriptionTemplateStatus previousStatus = data.getStatus();

        data.setDescription(model.getDescription());
        data.setLabel(model.getLabel());
        data.setTypeId(model.getType());
        data.setLanguage(model.getLanguage());
        data.setStatus(model.getStatus());
        data.setUpdatedAt(Instant.now());
        data.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(model.getDefinition())));
        if (isUpdate)
            this.entityManager.merge(data);
        else
            this.entityManager.persist(data);

        this.persistUsers(data.getId(), model.getUsers());
        if (this.conventionService.isListNullOrEmpty(model.getUsers()))
            this.addOwner(data);

        this.entityManager.flush();

        this.updateVersionStatusAndSave(data, previousStatus, data.getStatus());

        this.entityManager.flush();

        return this.builderFactory.builder(DescriptionTemplateBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermission).build(BaseFieldSet.build(fields, DescriptionTemplate._id), data);
    }

    private void updateVersionStatusAndSave(DescriptionTemplateEntity data, DescriptionTemplateStatus previousStatus, DescriptionTemplateStatus newStatus) {
        if (previousStatus.equals(newStatus))
            return;
        if (previousStatus.equals(DescriptionTemplateStatus.Finalized))
            throw new MyForbiddenException("Can not update finalized template");

        if (newStatus.equals(DescriptionTemplateStatus.Finalized)) {
            List<DescriptionTemplateEntity> latestVersionDescriptionTemplates = this.queryFactory.query(DescriptionTemplateQuery.class).versionStatuses(DescriptionTemplateVersionStatus.Current).isActive(IsActive.Active).groupIds(data.getGroupId()).collect();
            if (latestVersionDescriptionTemplates.size() > 1)
                throw new MyValidationException("Multiple previous template found");
            DescriptionTemplateEntity oldDescriptionTemplateEntity = latestVersionDescriptionTemplates.stream().findFirst().orElse(null);

            data.setVersionStatus(DescriptionTemplateVersionStatus.Current);

            if (oldDescriptionTemplateEntity != null) {
                data.setVersion((short) (oldDescriptionTemplateEntity.getVersion() + 1));

                oldDescriptionTemplateEntity.setVersionStatus(DescriptionTemplateVersionStatus.Previous);
                this.entityManager.merge(oldDescriptionTemplateEntity);
            } else {
                data.setVersion((short) 1);
            }
        }

    }

    private void persistUsers(UUID id, List<UserDescriptionTemplatePersist> users) throws InvalidApplicationException {
        if (users == null)
            users = new ArrayList<>();
        List<UserDescriptionTemplateEntity> items = this.queryFactory.query(UserDescriptionTemplateQuery.class).isActive(IsActive.Active).descriptionTemplateIds(id).collect();
        List<UUID> updatedCreatedIds = new ArrayList<>();
        for (UserDescriptionTemplatePersist user : users) {
            UserDescriptionTemplateEntity data = items.stream().filter(x -> x.getUserId().equals(user.getUserId()) && x.getRole().equals(user.getRole())).findFirst().orElse(null);
            if (data == null) {
                data = new UserDescriptionTemplateEntity();
                data.setId(UUID.randomUUID());
                data.setIsActive(IsActive.Active);
                data.setCreatedAt(Instant.now());
                data.setUpdatedAt(Instant.now());
                data.setDescriptionTemplateId(id);
                data.setUserId(user.getUserId());
                data.setRole(user.getRole());
                this.entityManager.persist(data);
                this.sendDescriptionTemplateInvitationEvent(data, NotificationContactType.EMAIL);
                this.sendDescriptionTemplateInvitationEvent(data, NotificationContactType.IN_APP);
            }
            updatedCreatedIds.add(data.getUserId());
        }
        List<UserDescriptionTemplateEntity> toDelete = items.stream().filter(x -> updatedCreatedIds.stream().noneMatch(y -> y.equals(x.getUserId()))).collect(Collectors.toList());

        this.deleterFactory.deleter(UserDescriptionTemplateDeleter.class).delete(toDelete);
    }

    private void sendDescriptionTemplateInvitationEvent(UserDescriptionTemplateEntity userDescriptionTemplate, NotificationContactType type) throws InvalidApplicationException {
        NotifyIntegrationEvent event = new NotifyIntegrationEvent();
        event.setTenantId(tenantScope.getTenant());
        event.setUserId(userScope.getUserIdSafe());

        UserEntity user = this.entityManager.find(UserEntity.class, userDescriptionTemplate.getUserId());
        DescriptionTemplateEntity descriptionTemplate = this.queryFactory.query(DescriptionTemplateQuery.class).isActive(IsActive.Active).ids(userDescriptionTemplate.getDescriptionTemplateId()).first();

        UserContactInfoQuery query = this.queryFactory.query(UserContactInfoQuery.class).userIds(user.getId());
        query.setOrder(new Ordering().addAscending(UserContactInfo._ordinal));

        List<ContactPair> contactPairs = new ArrayList<>();
        contactPairs.add(new ContactPair(ContactInfoType.Email, query.first().getValue()));
        NotificationContactData contactData = new NotificationContactData(contactPairs, null, null);
        event.setContactHint(jsonHandlingService.toJsonSafe(contactData));
        event.setContactTypeHint(type);
        event.setNotificationType(notificationProperties.getDescriptionTemplateInvitationType());
        NotificationFieldData data = new NotificationFieldData();
        List<FieldInfo> fieldInfoList = new ArrayList<>();
        fieldInfoList.add(new FieldInfo("{recipient}", DataType.String, user.getName()));
        fieldInfoList.add(new FieldInfo("{templateName}", DataType.String, descriptionTemplate.getLabel()));
        fieldInfoList.add(new FieldInfo("{templateID}", DataType.String, descriptionTemplate.getId().toString()));
        data.setFields(fieldInfoList);
        event.setData(jsonHandlingService.toJsonSafe(data));
        eventHandler.handle(event);
    }

    private void addOwner(DescriptionTemplateEntity descriptionTemplateEntity) throws InvalidApplicationException {
        UserDescriptionTemplateEntity data = new UserDescriptionTemplateEntity();
        data.setId(UUID.randomUUID());
        data.setIsActive(IsActive.Active);
        data.setCreatedAt(Instant.now());
        data.setUpdatedAt(Instant.now());
        data.setRole(UserDescriptionTemplateRole.Owner);
        data.setUserId(userScope.getUserId());
        data.setDescriptionTemplateId(descriptionTemplateEntity.getId());
        this.entityManager.persist(data);
    }

    private @NotNull DefinitionEntity buildDefinitionEntity(DefinitionPersist persist) {
        DefinitionEntity data = new DefinitionEntity();
        if (persist == null)
            return data;

        if (!this.conventionService.isListNullOrEmpty(persist.getPages())) {
            data.setPages(new ArrayList<>());
            for (PagePersist pagePersist : persist.getPages()) {
                data.getPages().add(this.buildPageEntity(pagePersist));
            }
        }

        return data;
    }

    private @NotNull SectionEntity buildSectionEntity(SectionPersist persist) {
        SectionEntity data = new SectionEntity();
        if (persist == null)
            return data;

        data.setId(persist.getId());
        data.setDescription(persist.getDescription());
        data.setOrdinal(persist.getOrdinal());
        data.setDefaultVisibility(persist.getDefaultVisibility());
        data.setMultiplicity(persist.getMultiplicity());
        data.setTitle(persist.getTitle());

        if (!this.conventionService.isListNullOrEmpty(persist.getSections())) {
            data.setSections(new ArrayList<>());
            for (SectionPersist sectionPersist : persist.getSections()) {
                data.getSections().add(this.buildSectionEntity(sectionPersist));
            }
        }

        if (!this.conventionService.isListNullOrEmpty(persist.getFieldSets())) {
            data.setFieldSets(new ArrayList<>());
            for (FieldSetPersist fieldSetPersist : persist.getFieldSets()) {
                data.getFieldSets().add(this.buildFieldSetEntity(fieldSetPersist));
            }
        }

        return data;
    }

    private @NotNull FieldSetEntity buildFieldSetEntity(FieldSetPersist persist) {
        FieldSetEntity data = new FieldSetEntity();
        if (persist == null)
            return data;

        data.setId(persist.getId());
        data.setDescription(persist.getDescription());
        data.setOrdinal(persist.getOrdinal());
        data.setTitle(persist.getTitle());
        data.setNumbering(persist.getNumbering());
        data.setAdditionalInformation(persist.getAdditionalInformation());
        data.setExtendedDescription(persist.getExtendedDescription());
        if (persist.getMultiplicity() != null)
            data.setMultiplicity(this.buildMultiplicityEntity(persist.getMultiplicity()));
        data.setHasCommentField(persist.getHasCommentField());

        if (!this.conventionService.isListNullOrEmpty(persist.getFields())) {
            data.setFields(new ArrayList<>());
            for (FieldPersist fieldPersist : persist.getFields()) {
                data.getFields().add(this.buildFieldEntity(fieldPersist));
            }
        }
        return data;
    }

    private @NotNull FieldEntity buildFieldEntity(FieldPersist persist) {
        FieldEntity data = new FieldEntity();
        if (persist == null)
            return data;

        data.setId(persist.getId());
        data.setOrdinal(persist.getOrdinal());
        data.setSchematics(persist.getSchematics());
        //data.setNumbering(persist.get()); //TODO
        data.setDefaultValue(persist.getDefaultValue());
        data.setValidations(persist.getValidations());
        data.setIncludeInExport(persist.getIncludeInExport());
        if (persist.getData() != null)
            data.setData(this.buildFieldDataEntity(persist.getData()));

        if (!this.conventionService.isListNullOrEmpty(persist.getVisibilityRules())) {
            data.setVisibilityRules(new ArrayList<>());
            for (RulePersist fieldPersist : persist.getVisibilityRules()) {
                data.getVisibilityRules().add(this.buildRuleEntity(fieldPersist, data));
            }
        }
        return data;
    }

    private BaseFieldDataEntity buildFieldDataEntity(BaseFieldDataPersist persist) {
        if (persist == null)
            return null;
        return this.fieldDataHelperServiceProvider.get(persist.getFieldType()).applyPersist(persist);
    }

    private @NotNull RuleEntity buildRuleEntity(RulePersist persist, FieldEntity fieldEntity){
        FieldType fieldType = fieldEntity != null && fieldEntity.getData() != null ? fieldEntity.getData().getFieldType() :  FieldType.FREE_TEXT;
        RuleEntity data = new RuleEntity();
        if (persist == null) return data;
        data.setTarget(persist.getTarget());

        if (FieldType.isTextType(fieldType)) {
            if (FieldType.UPLOAD.equals(fieldType) && !this.conventionService.isNullOrEmpty(persist.getTextValue())){
                throw new NotSupportedException("Upload not supported");
            } else {
                data.setTextValue(persist.getTextValue());
            }
        }
        else if (FieldType.isTextListType(fieldType)) {
            if (FieldType.INTERNAL_ENTRIES_DMPS.equals(fieldType) && !this.conventionService.isListNullOrEmpty(persist.getTextListValue())){
                List<UUID> ids = persist.getTextListValue().stream().map(UUID::fromString).toList();
                Set<UUID> existingIds = this.queryFactory.query(DmpQuery.class).ids(ids).isActive(IsActive.Active).collectAs(new BaseFieldSet().ensure(Dmp._id)).stream().map(DmpEntity::getId).collect(Collectors.toSet());
                for (UUID id : ids){
                    if (!existingIds.contains(id)) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, Dmp.class.getSimpleName()}, LocaleContextHolder.getLocale()));
                }
            } if (FieldType.INTERNAL_ENTRIES_DESCRIPTIONS.equals(fieldType) && !this.conventionService.isListNullOrEmpty(persist.getTextListValue())){
                List<UUID> ids = persist.getTextListValue().stream().map(UUID::fromString).toList();
                Set<UUID> existingIds = this.queryFactory.query(DescriptionQuery.class).ids(ids).isActive(IsActive.Active).collectAs(new BaseFieldSet().ensure(Description._id)).stream().map(DescriptionEntity::getId).collect(Collectors.toSet());
                for (UUID id : ids){
                    if (!existingIds.contains(id)) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, Description.class.getSimpleName()}, LocaleContextHolder.getLocale()));
                }
            }
            data.setTextListValue(persist.getTextListValue());
        }
        else if (FieldType.isReferenceType(fieldType) ) {
            List<UUID> referenceIds = this.patchAndSaveReferences(persist.getReferences(), fieldEntity);
            if (!referenceIds.isEmpty()) data.setTextListValue(referenceIds.stream().map(UUID::toString).toList());
        }
        else if (FieldType.isDateType(fieldType))  data.setDateValue(persist.getDateValue());
        else if (FieldType.isExternalIdentifierType(fieldType) && persist.getExternalIdentifier() != null)  data.setExternalIdentifier(this.buildExternalIdentifierEntity(persist.getExternalIdentifier()));

        return data;
    }
    private @NotNull ExternalIdentifierEntity buildExternalIdentifierEntity(ExternalIdentifierPersist persist){
        ExternalIdentifierEntity data = new ExternalIdentifierEntity();
        if (persist == null) return data;

        data.setIdentifier(persist.getIdentifier());
        data.setType(persist.getType());
        return data;
    }

    private List<UUID> patchAndSaveReferences(List<ReferencePersist> models, FieldEntity fieldEntity) {
        if (models == null) models = new ArrayList<>();

        List<UUID> referenceIds = new ArrayList<>();

        for (ReferencePersist model : models) {
            ReferenceEntity data;
            if (this.conventionService.isValidGuid(model.getId())){
                data = this.entityManager.find(ReferenceEntity.class, model.getId());
                if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), Reference.class.getSimpleName()}, LocaleContextHolder.getLocale()));
            } else {
                ReferenceTypeDataEntity referenceTypeDataEntity =  ((ReferenceTypeDataEntity)fieldEntity.getData());
                data = this.queryFactory.query(ReferenceQuery.class).sourceTypes(model.getSourceType()).typeIds(referenceTypeDataEntity.getReferenceTypeId()).sources(model.getSource()).isActive(IsActive.Active).references(model.getReference()).first();
                if (data == null){
                    data = new ReferenceEntity();
                    data.setId(UUID.randomUUID());
                    data.setIsActive(IsActive.Active);
                    data.setCreatedAt(Instant.now());
                    data.setTypeId(referenceTypeDataEntity.getReferenceTypeId());

                    data.setDefinition(this.xmlHandlingService.toXmlSafe(this.buildReferenceDefinitionEntity(model.getDefinition())));
                    data.setUpdatedAt(Instant.now());
                    data.setReference(model.getReference());
                    data.setAbbreviation(model.getAbbreviation());
                    data.setSource(model.getSource());
                    data.setSourceType(model.getSourceType());
                    this.entityManager.persist(data);
                }
            }
            referenceIds.add(data.getId());
        }
        this.entityManager.flush();

        return referenceIds;
    }

    private @NotNull eu.eudat.commons.types.reference.DefinitionEntity buildReferenceDefinitionEntity(eu.eudat.model.persist.referencedefinition.DefinitionPersist persist){
        eu.eudat.commons.types.reference.DefinitionEntity data = new eu.eudat.commons.types.reference.DefinitionEntity();
        if (persist == null) return data;
        if (!this.conventionService.isListNullOrEmpty(persist.getFields())){
            data.setFields(new ArrayList<>());
            for (eu.eudat.model.persist.referencedefinition.FieldPersist fieldPersist: persist.getFields()) {
                data.getFields().add(this.buildRefernceFieldEntity(fieldPersist));
            }
        }

        return data;
    }

    private @NotNull eu.eudat.commons.types.reference.FieldEntity buildRefernceFieldEntity(eu.eudat.model.persist.referencedefinition.FieldPersist persist){
        eu.eudat.commons.types.reference.FieldEntity data = new eu.eudat.commons.types.reference.FieldEntity();
        if (persist == null) return data;

        data.setCode(persist.getCode());
        data.setDataType(persist.getDataType());
        data.setCode(persist.getCode());

        return data;
    }

    private @NotNull MultiplicityEntity buildMultiplicityEntity(MultiplicityPersist persist) {
        MultiplicityEntity data = new MultiplicityEntity();
        if (persist == null)
            return data;

        if (persist.getMax() != null)
            data.setMax(persist.getMax());
        if (persist.getMin() != null)
            data.setMin(persist.getMin());
        if (persist.getPlaceholder() != null)
            data.setPlaceholder(persist.getPlaceholder());
        if (persist.getTableView() != null)
            data.setTableView(persist.getTableView());
        return data;
    }

    private @NotNull PageEntity buildPageEntity(PagePersist persist) {
        PageEntity data = new PageEntity();
        if (persist == null)
            return data;

        data.setId(persist.getId());
        data.setOrdinal(persist.getOrdinal());
        data.setTitle(persist.getTitle());

        if (!this.conventionService.isListNullOrEmpty(persist.getSections())) {
            data.setSections(new ArrayList<>());
            for (SectionPersist sectionPersist : persist.getSections()) {
                data.getSections().add(this.buildSectionEntity(sectionPersist));
            }
        }
        return data;
    }

    //endregion 

    //region Delete

    public void deleteAndSave(UUID id) throws MyForbiddenException, InvalidApplicationException {
        logger.debug("deleting dataset: {}", id);

        this.authorizationService.authorizeForce(Permission.DeleteDescriptionTemplate);

        DescriptionTemplateEntity data = this.entityManager.find(DescriptionTemplateEntity.class, id);
        if (data == null) throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        if (data.getVersionStatus().equals(DescriptionTemplateVersionStatus.Current)){
            DescriptionTemplateQuery descriptionTemplateQuery = this.queryFactory.query(DescriptionTemplateQuery.class)
                    .statuses(DescriptionTemplateStatus.Finalized)
                    .excludedIds(data.getId())
                    .isActive(IsActive.Active)
                    .groupIds(data.getGroupId());

            descriptionTemplateQuery.setOrder(new Ordering().addDescending(DescriptionTemplate._version));
            DescriptionTemplateEntity previousFinalized = descriptionTemplateQuery.first();
            if (previousFinalized != null){
                previousFinalized.setVersionStatus(DescriptionTemplateVersionStatus.Current);
                this.entityManager.merge(previousFinalized);
            }
            data.setVersionStatus(DescriptionTemplateVersionStatus.NotFinalized);
            this.entityManager.merge(data);
            this.entityManager.flush();
        }
        
        this.deleterFactory.deleter(DescriptionTemplateDeleter.class).deleteAndSaveByIds(List.of(id));
    }

    //endregion 

    //region Clone

    public DescriptionTemplate buildClone(UUID id, FieldSet fields) throws MyForbiddenException, MyValidationException, MyApplicationException, MyNotFoundException {
        logger.debug(new MapLogEntry("persisting data").And("id", id).And("fields", fields));

        this.authorizationService.authorizeForce(Permission.CloneDescriptionTemplate);

        DescriptionTemplateQuery query = this.queryFactory.query(DescriptionTemplateQuery.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermission).ids(id);
        DescriptionTemplate model = this.builderFactory.builder(DescriptionTemplateBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermission).build(fields, query.firstAs(fields));
        if (model == null)
            throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        model.setLabel(model.getLabel() + " new ");
        model.setId(null);
        model.setHash(null);
        model.setStatus(DescriptionTemplateStatus.Draft);
        this.reassignDefinition(model.getDefinition());

        return model;
    }

    private void reassignDefinition(Definition model) {
        if (model == null)
            return;

        if (model.getPages() != null) {
            for (Page page : model.getPages()) {
                this.reassignPage(page);
            }
        }
    }

    private void reassignPage(Page model) {
        if (model == null)
            return;
        model.setId(UUID.randomUUID().toString());

        if (model.getSections() != null) {
            for (Section section : model.getSections()) {
                this.reassignSection(section);
            }
        }

    }

    private void reassignSection(Section model) {
        if (model == null)
            return;
        model.setId(UUID.randomUUID().toString());

        if (model.getSections() != null) {
            for (Section section : model.getSections()) {
                this.reassignSection(section);
            }
        }
        if (model.getFieldSets() != null) {
            for (eu.eudat.model.descriptiontemplatedefinition.FieldSet fieldSet : model.getFieldSets()) {
                this.reassignFieldSet(fieldSet);
            }
        }
    }

    private void reassignFieldSet(eu.eudat.model.descriptiontemplatedefinition.FieldSet model) {
        if (model == null)
            return;
        model.setId(UUID.randomUUID().toString());

        if (model.getFields() != null) {
            for (Field field : model.getFields()) {
                this.reassignField(field);
            }
        }
    }

    private void reassignField(Field model) {
        if (model == null)
            return;
        model.setId(UUID.randomUUID().toString());
    }

    //endregion

    //region NewVersion

    public DescriptionTemplate createNewVersion(NewVersionDescriptionTemplatePersist model, FieldSet fields) throws MyForbiddenException, MyValidationException, MyApplicationException, MyNotFoundException, InvalidApplicationException, JAXBException {
        logger.debug(new MapLogEntry("persisting data descriptionTemplate").And("model", model).And("fields", fields));

        this.authorizationService.authorizeForce(Permission.CreateNewVersionDescriptionTemplate);

        DescriptionTemplateEntity oldDescriptionTemplateEntity = this.entityManager.find(DescriptionTemplateEntity.class, model.getId());
        if (oldDescriptionTemplateEntity == null)
            throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{model.getId(), DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));
        if (!this.conventionService.hashValue(oldDescriptionTemplateEntity.getUpdatedAt()).equals(model.getHash()))
            throw new MyValidationException(this.errors.getHashConflict().getCode(), this.errors.getHashConflict().getMessage());

        List<DescriptionTemplateEntity> latestVersionDescriptionTemplates = this.queryFactory.query(DescriptionTemplateQuery.class)
                .versionStatuses(DescriptionTemplateVersionStatus.Current)
                .isActive(IsActive.Active)
                .groupIds(oldDescriptionTemplateEntity.getGroupId())
                .collect();
        if (latestVersionDescriptionTemplates.isEmpty())
            throw new MyValidationException("Previous template not found");
        if (latestVersionDescriptionTemplates.size() > 1)
            throw new MyValidationException("Multiple previous template found");
        if (!latestVersionDescriptionTemplates.getFirst().getVersion().equals(oldDescriptionTemplateEntity.getVersion()))
            throw new MyValidationException(this.errors.getDescriptionTemplateNewVersionConflict().getCode(), this.errors.getDescriptionTemplateNewVersionConflict().getMessage());
        Long notFinalizedCount = this.queryFactory.query(DescriptionTemplateQuery.class)
                .versionStatuses(DescriptionTemplateVersionStatus.NotFinalized)
                .groupIds(oldDescriptionTemplateEntity.getGroupId())
                .isActive(IsActive.Active)
                .count();
        if (notFinalizedCount > 0)
            throw new MyValidationException("Already created draft for this template");

        DescriptionTemplateEntity data = new DescriptionTemplateEntity();
        data.setId(UUID.randomUUID());
        data.setIsActive(IsActive.Active);
        data.setCreatedAt(Instant.now());
        data.setUpdatedAt(Instant.now());
        data.setVersionStatus(DescriptionTemplateVersionStatus.NotFinalized);
        data.setGroupId(oldDescriptionTemplateEntity.getGroupId());
        data.setVersion((short) (oldDescriptionTemplateEntity.getVersion() + 1));
        data.setDescription(model.getDescription());
        data.setLabel(model.getLabel());
        data.setTypeId(model.getType());
        data.setLanguage(model.getLanguage());
        data.setStatus(model.getStatus());
        data.setDefinition(this.xmlHandlingService.toXml(this.buildDefinitionEntity(model.getDefinition())));

        this.entityManager.persist(data);

        this.persistUsers(data.getId(), model.getUsers());
        //this.addOwner(data);

        this.entityManager.flush();

        this.updateVersionStatusAndSave(data, DescriptionTemplateStatus.Draft, data.getStatus());

        this.entityManager.flush();

        return this.builderFactory.builder(DescriptionTemplateBuilder.class).authorize(AuthorizationFlags.OwnerOrDmpAssociatedOrPermission).build(BaseFieldSet.build(fields, DescriptionTemplate._id), data);
    }

    //endregion

    //region Import

    public DescriptionTemplate importXml(byte[] bytes, UUID id, String label, FieldSet fields) throws MyForbiddenException, MyNotFoundException, JAXBException, ParserConfigurationException, TransformerException, InvalidApplicationException, IOException, InstantiationException, IllegalAccessException, SAXException {
        logger.debug(new MapLogEntry("import data").And("bytes", bytes).And("fields", fields));

        this.authorizationService.authorizeForce(Permission.ImportDescriptionTemplate);

        DescriptionTemplateImportExport importXml = this.xmlHandlingService.fromXml(DescriptionTemplateImportExport.class, new String(bytes, StandardCharsets.UTF_8));
        if (id == null) {
            DescriptionTemplatePersist persist = new DescriptionTemplatePersist();
            persist.setLabel(label);
            persist.setStatus(DescriptionTemplateStatus.Draft);
            persist.setDescription(importXml.getDescription());
            persist.setLanguage(importXml.getLanguage());
            persist.setType(importXml.getType());
            persist.setDefinition(this.xmlDefinitionToPersist(importXml));
            this.validatorFactory.validator(DescriptionTemplatePersist.DescriptionTemplatePersistValidator.class).validateForce(persist);
            return this.persist(persist, fields);
        } else {
            NewVersionDescriptionTemplatePersist persist = new NewVersionDescriptionTemplatePersist();
            persist.setId(id);
            persist.setLabel(label);
            persist.setStatus(DescriptionTemplateStatus.Draft);
            persist.setDescription(importXml.getDescription());
            persist.setLanguage(importXml.getLanguage());
            persist.setDefinition(this.xmlDefinitionToPersist(importXml));
            persist.setType(importXml.getType());

            DescriptionTemplateEntity oldDescriptionTemplateEntity = this.entityManager.find(DescriptionTemplateEntity.class, id);
            if (oldDescriptionTemplateEntity == null)
                throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));
            persist.setHash(this.conventionService.hashValue(oldDescriptionTemplateEntity.getUpdatedAt()));
            this.validatorFactory.validator(DescriptionTemplatePersist.DescriptionTemplatePersistValidator.class).validateForce(persist);
            return this.createNewVersion(persist, fields);
        }
    }

    public DefinitionPersist xmlDefinitionToPersist(DescriptionTemplateImportExport importExport) {
        DefinitionPersist definitionPersist = new DefinitionPersist();
        if (importExport == null)
            return null;

        List<PagePersist> pagesDatasetEntity = new LinkedList<>();
        if (!this.conventionService.isListNullOrEmpty(importExport.getPages())) {
            for (PageImportExport xmlPage : importExport.getPages()) {
                pagesDatasetEntity.add(this.xmlPageToPersist(xmlPage));
            }
        }
        definitionPersist.setPages(pagesDatasetEntity);

        return definitionPersist;
    }

    public PagePersist xmlPageToPersist(PageImportExport importExport) {
        PagePersist pageEntity = new PagePersist();
        pageEntity.setId(importExport.getId());
        pageEntity.setOrdinal(importExport.getOrdinal());
        pageEntity.setTitle(importExport.getTitle());
        if (!this.conventionService.isListNullOrEmpty(importExport.getSections())) {
            List<SectionPersist> sectionsListEntity = new LinkedList<>();
            for (SectionImportExport xmlSection : importExport.getSections()) {
                sectionsListEntity.add(this.xmlSectionToPersist(xmlSection));
            }
            pageEntity.setSections(sectionsListEntity);

        }
        return pageEntity;
    }

    public SectionPersist xmlSectionToPersist(SectionImportExport importExport) {
        SectionPersist sectionEntity = new SectionPersist();
        List<SectionPersist> sectionsListEntity = new LinkedList<>();

        if (!this.conventionService.isListNullOrEmpty(importExport.getSections())) {
            for (SectionImportExport xmlSection : importExport.getSections()) {
                sectionsListEntity.add(this.xmlSectionToPersist(xmlSection));
            }
        }
        sectionEntity.setId(importExport.getId());
        sectionEntity.setOrdinal(importExport.getOrdinal());
        sectionEntity.setTitle(importExport.getTitle());
        sectionEntity.setDescription(importExport.getDescription());
        List<FieldSetPersist> fieldSetEntity = new LinkedList<>();
        if (!this.conventionService.isListNullOrEmpty(importExport.getFieldSets())) {
            for (FieldSetImportExport xmlFieldSet : importExport.getFieldSets()) {
                fieldSetEntity.add(this.toFieldSetModel(xmlFieldSet));
            }
        }
        sectionEntity.setFieldSets(fieldSetEntity);

        sectionEntity.setSections(sectionsListEntity);
        sectionEntity.setDefaultVisibility(importExport.getDefaultVisibility());
        return sectionEntity;
    }

    public FieldSetPersist toFieldSetModel(FieldSetImportExport importExport) {
        FieldSetPersist fieldSet1Entity = new FieldSetPersist();
        fieldSet1Entity.setId(importExport.getId());
        fieldSet1Entity.setOrdinal(importExport.getOrdinal());
        fieldSet1Entity.setHasCommentField(importExport.getHasCommentField() != null ? importExport.getHasCommentField() : false);
        fieldSet1Entity.setMultiplicity(importExport.getMultiplicity() != null ? this.xmlMultiplicityToPersist(importExport.getMultiplicity()) : null);
        fieldSet1Entity.setTitle(importExport.getTitle());
        fieldSet1Entity.setDescription(importExport.getDescription());
        fieldSet1Entity.setExtendedDescription(importExport.getExtendedDescription());
        fieldSet1Entity.setAdditionalInformation(importExport.getAdditionalInformation());

        List<FieldPersist> fieldsEntity = new LinkedList<>();
        if (!this.conventionService.isListNullOrEmpty(importExport.getFields())) {
            for (FieldImportExport xmlField : importExport.getFields()) {
                fieldsEntity.add(this.xmlFieldToPersist(xmlField));
            }
        }
        fieldSet1Entity.setFields(fieldsEntity);
        return fieldSet1Entity;
    }

    public FieldPersist xmlFieldToPersist(FieldImportExport importExport) {
        FieldPersist fieldEntity = new FieldPersist();
        fieldEntity.setId(importExport.getId());
        fieldEntity.setOrdinal(importExport.getOrdinal());
        fieldEntity.setValidations(importExport.getValidations());
        fieldEntity.setDefaultValue(importExport.getDefaultValue());
        List<RulePersist> rulePersists = new ArrayList<>();
        if (importExport.getVisibilityRules() != null) {
            for (RuleImportExport xmlRule : importExport.getVisibilityRules()) {
                rulePersists.add(this.toRuleModel(xmlRule));
            }
        }
        fieldEntity.setVisibilityRules(rulePersists);
        FieldType fieldType = importExport.getFieldType();

        if (importExport.getData() != null) {
            FieldDataHelperService fieldDataHelperService = this.fieldDataHelperServiceProvider.get(fieldType);
            fieldEntity.setData(fieldDataHelperService.importExportMapDataToPersist(importExport.getData()));
        }
        fieldEntity.setSchematics(importExport.getSchematics());
        return fieldEntity;
    }

    public RulePersist toRuleModel(RuleImportExport importExport) {
        RulePersist ruleEntity = new RulePersist();
        ruleEntity.setTarget(importExport.getTarget());
        ruleEntity.setDateValue(importExport.getDateValue());
        //ruleEntity.setReferences(importExport.get()); //TODO
        ruleEntity.setTextValue(importExport.getTextValue());
        ruleEntity.setTextListValue(importExport.getTextListValue());
        return ruleEntity;
    }

    public MultiplicityPersist xmlMultiplicityToPersist(MultiplicityImportExport importXml) {
        MultiplicityPersist multiplicityEntity = new MultiplicityPersist();
        multiplicityEntity.setMax(importXml.getMax());
        multiplicityEntity.setMin(importXml.getMin());
        multiplicityEntity.setPlaceholder(importXml.getPlaceholder());
        multiplicityEntity.setTableView(importXml.getTableView());
        return multiplicityEntity;
    }

    //endregion

    //region Export

    @Override
    public ResponseEntity<byte[]> exportXml(UUID id) throws MyForbiddenException, MyNotFoundException, JAXBException, ParserConfigurationException, IOException, InstantiationException, IllegalAccessException, SAXException {
        logger.debug(new MapLogEntry("persisting data").And("id", id));

        this.authorizationService.authorizeForce(Permission.ExportDescriptionTemplate);
        DescriptionTemplateEntity data = this.entityManager.find(DescriptionTemplateEntity.class, id);
        if (data == null)
            throw new MyNotFoundException(messageSource.getMessage("General_ItemNotFound", new Object[]{id, DescriptionTemplate.class.getSimpleName()}, LocaleContextHolder.getLocale()));

        DefinitionEntity definition = this.xmlHandlingService.fromXml(DefinitionEntity.class, data.getDefinition());
        String xml = this.xmlHandlingService.toXml(this.definitionXmlToExport(data, definition));
        return this.responseUtilsService.buildResponseFileFromText(xml, data.getLabel() + ".xml");
    }

    private DescriptionTemplateImportExport definitionXmlToExport(DescriptionTemplateEntity data, DefinitionEntity entity) {
        DescriptionTemplateImportExport xml = new DescriptionTemplateImportExport();
        xml.setType(data.getTypeId());
        xml.setLanguage(data.getLanguage());
        xml.setDescription(data.getDescription());
        List<PageImportExport> pagesDatasetEntity = new LinkedList<>();
        for (PageEntity xmlPage : entity.getPages()) {
            pagesDatasetEntity.add(this.pageXmlToExport(xmlPage));
        }
        xml.setPages(pagesDatasetEntity);

        return xml;
    }

    private PageImportExport pageXmlToExport(PageEntity entity) {
        PageImportExport xml = new PageImportExport();
        xml.setId(entity.getId());
        xml.setOrdinal(entity.getOrdinal());
        xml.setTitle(entity.getTitle());
        List<SectionImportExport> sectionsListEntity = new LinkedList<>();
        if (!this.conventionService.isListNullOrEmpty(entity.getSections())) {
            for (SectionEntity section : entity.getSections()) {
                sectionsListEntity.add(this.sectionXmlToExport(section));
            }
        }
        xml.setSections(sectionsListEntity);

        return xml;
    }

    private SectionImportExport sectionXmlToExport(SectionEntity entity) {
        SectionImportExport xml = new SectionImportExport();
        List<SectionImportExport> sectionsListEntity = new LinkedList<>();

        if (!this.conventionService.isListNullOrEmpty(entity.getSections())) {
            for (SectionEntity xmlSection : entity.getSections()) {
                sectionsListEntity.add(this.sectionXmlToExport(xmlSection));
            }
        }
        xml.setSections(sectionsListEntity);

        xml.setId(entity.getId());
        xml.setOrdinal(entity.getOrdinal());
        xml.setTitle(entity.getTitle());
        xml.setDescription(entity.getDescription());
        List<FieldSetImportExport> fieldSetEntity = new LinkedList<>();
        if (!this.conventionService.isListNullOrEmpty(entity.getFieldSets())) {
            for (FieldSetEntity xmlFieldSet : entity.getFieldSets()) {
                fieldSetEntity.add(this.fieldSetXmlToExport(xmlFieldSet));
            }
        }
        xml.setFieldSets(fieldSetEntity);

        xml.setDefaultVisibility(entity.isDefaultVisibility());
        return xml;
    }

    private FieldSetImportExport fieldSetXmlToExport(FieldSetEntity entity) {
        FieldSetImportExport fieldSet1Entity = new FieldSetImportExport();
        fieldSet1Entity.setId(entity.getId());
        fieldSet1Entity.setOrdinal(entity.getOrdinal());
        fieldSet1Entity.setHasCommentField(entity.getHasCommentField());
        fieldSet1Entity.setMultiplicity(entity.getMultiplicity() != null ? this.multiplicityXmlToExport(entity.getMultiplicity()) : null);
        fieldSet1Entity.setTitle(entity.getTitle());
        fieldSet1Entity.setDescription(entity.getDescription());
        fieldSet1Entity.setExtendedDescription(entity.getExtendedDescription());
        fieldSet1Entity.setAdditionalInformation(entity.getAdditionalInformation());

        List<FieldImportExport> fieldsEntity = new LinkedList<>();
        if (entity.getFields() != null) {
            for (FieldEntity xmlField : entity.getFields()) {
                fieldsEntity.add(this.fieldXmlToExport(xmlField));
            }
        }
        fieldSet1Entity.setFields(fieldsEntity);
        return fieldSet1Entity;
    }

    private FieldImportExport fieldXmlToExport(FieldEntity entity) {
        FieldImportExport xml = new FieldImportExport();
        xml.setId(entity.getId());
        xml.setOrdinal(entity.getOrdinal());
        xml.setValidations(entity.getValidations());
        xml.setDefaultValue(entity.getDefaultValue());
        List<RuleImportExport> rulePersists = new ArrayList<>();
        if (!this.conventionService.isListNullOrEmpty(entity.getVisibilityRules())) {
            for (RuleEntity xmlRule : entity.getVisibilityRules()) {
                rulePersists.add(this.toRuleModel(xmlRule));
            }
        }
        xml.setVisibilityRules(rulePersists);

        if (entity.getData() != null) {
            FieldType fieldType = entity.getData().getFieldType();
            FieldDataHelperService fieldDataHelperService = this.fieldDataHelperServiceProvider.get(fieldType);
            xml.setData(fieldDataHelperService.dataToImportExportXml(entity.getData()));
        }
        xml.setSchematics(entity.getSchematics());
        return xml;
    }

    private RuleImportExport toRuleModel(RuleEntity entity) {
        RuleImportExport xml = new RuleImportExport();
        xml.setTarget(entity.getTarget());
        xml.setDateValue(entity.getDateValue());
        xml.setTextListValue(entity.getTextListValue());
        xml.setTextValue(entity.getTextValue());
        if (entity.getExternalIdentifier() != null){
            xml.setExternalIdentifier(new ExternalIdentifierImportExport());
            xml.getExternalIdentifier().setIdentifier(entity.getExternalIdentifier().getIdentifier());
            xml.getExternalIdentifier().setType(entity.getExternalIdentifier().getType());
        } 
        return xml;
    }

    private MultiplicityImportExport multiplicityXmlToExport(MultiplicityEntity entity) {
        MultiplicityImportExport xml = new MultiplicityImportExport();
        xml.setMax(entity.getMax());
        xml.setMin(entity.getMin());
        xml.setPlaceholder(entity.getPlaceholder());
        xml.setTableView(entity.getTableView());
        return xml;
    }

    //endregion
}