package eu.eudat.logic.managers;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.authorization.Permission;
import eu.eudat.commons.enums.DescriptionStatus;
import eu.eudat.commons.enums.DmpAccessType;
import eu.eudat.commons.enums.DmpStatus;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.scope.user.UserScope;
import eu.eudat.data.*;
import eu.eudat.data.dao.criteria.*;
import eu.eudat.data.dao.entities.DatasetDao;
import eu.eudat.data.old.*;
import eu.eudat.data.query.items.table.dataset.DatasetPublicTableRequest;
import eu.eudat.data.query.items.table.dataset.DatasetTableRequest;
import eu.eudat.data.query.items.table.datasetprofile.DatasetProfileTableRequestItem;
import eu.eudat.exceptions.security.ForbiddenException;
import eu.eudat.exceptions.security.UnauthorisedException;
import eu.eudat.logic.builders.BuilderFactory;
import eu.eudat.logic.builders.entity.UserInfoBuilder;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.forms.VisibilityRuleService;
import eu.eudat.logic.services.forms.VisibilityRuleServiceImpl;
import eu.eudat.logic.services.operations.DatabaseRepository;
import eu.eudat.model.file.FileEnvelope;
import eu.eudat.logic.utilities.documents.types.ParagraphStyle;
import eu.eudat.logic.utilities.documents.word.WordBuilder;
import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder;
import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.HintedModelFactory;
import eu.eudat.models.data.dataset.DatasetOverviewModel;
import eu.eudat.models.data.datasetImport.DatasetImportField;
import eu.eudat.models.data.datasetImport.DatasetImportPagedDatasetProfile;
import eu.eudat.models.data.datasetprofile.DatasetProfileListingModel;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.dmp.AssociatedProfile;
import eu.eudat.models.data.helpers.common.DataTableData;
import eu.eudat.models.data.listingmodels.DatasetListingModel;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import eu.eudat.query.DescriptionTemplateQuery;
import eu.eudat.query.DmpDescriptionTemplateQuery;
import eu.eudat.query.DmpUserQuery;
import eu.eudat.queryable.QueryableList;
import eu.eudat.types.MetricNames;
import gr.cite.commons.web.authz.service.AuthorizationService;
import gr.cite.tools.data.query.QueryFactory;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import jakarta.activation.MimetypesFileTypeMap;
import jakarta.persistence.criteria.Join;
import jakarta.persistence.criteria.JoinType;
import jakarta.transaction.Transactional;
import jakarta.xml.bind.JAXBContext;
import jakarta.xml.bind.JAXBException;
import jakarta.xml.bind.Unmarshaller;

import javax.management.InvalidApplicationException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import java.io.*;
import java.math.BigInteger;
import java.nio.file.Files;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Component
public class DatasetManager {
    private static final Logger logger = LoggerFactory.getLogger(DatasetManager.class);

//    private final Map<NotificationType, String> notificationPaths = Stream.of(new Object[][] {
//            {NotificationType.DATASET_MODIFIED, "/datasets/edit"},
//            {NotificationType.DATASET_MODIFIED_FINALISED, "/datasets/edit"}
//    }).collect(Collectors.toMap(data -> (NotificationType) data[0], data -> (String) data[1]));

    private ApiContext apiContext;
    private DatabaseRepository databaseRepository;
//    private DatasetRepository datasetRepository;
    private BuilderFactory builderFactory;
    private ConfigLoader configLoader;
    private Environment environment;
    private final MetricsManager metricsManager;
    private final FileManager fileManager;
    private final UserScope userScope;
    private final AuthorizationService authorizationService;
    private final QueryFactory queryFactory;

    @Autowired
    public DatasetManager(ApiContext apiContext, ConfigLoader configLoader, Environment environment, MetricsManager metricsManager,
                          FileManager fileManager, UserScope userScope, AuthorizationService authorizationService, QueryFactory queryFactory) {
        this.apiContext = apiContext;
        this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
//        this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository();
        this.builderFactory = apiContext.getOperationsContext().getBuilderFactory();
        this.configLoader = configLoader;
        this.environment = environment;
        this.metricsManager = metricsManager;
        this.fileManager = fileManager;
        this.userScope = userScope;
        this.authorizationService = authorizationService;
        this.queryFactory = queryFactory;
    }

    public DataTableData<DatasetListingModel> getPaged(DatasetTableRequest datasetTableRequest) throws Exception {
        //TODO
//        DatasetCriteria datasetCriteria = new DatasetCriteria();
//        datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
//        datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetTemplates());
//        if (datasetTableRequest.getCriteria().getStatus() != null) {
//            datasetCriteria.setStatus(datasetTableRequest.getCriteria().getStatus().shortValue());
//        }
//        datasetCriteria.setDmps(datasetTableRequest.getCriteria().getDmpIds());
//        datasetCriteria.setGroupIds(datasetTableRequest.getCriteria().getGroupIds());
//        datasetCriteria.setGrants(datasetTableRequest.getCriteria().getGrants());
//        datasetCriteria.setCollaborators(datasetTableRequest.getCriteria().getCollaborators());
//        datasetCriteria.setAllowAllVersions(datasetTableRequest.getCriteria().getAllVersions());
//        datasetCriteria.setOrganiztions(datasetTableRequest.getCriteria().getOrganisations());
//        datasetCriteria.setTags(datasetTableRequest.getCriteria().getTags());
//        if (datasetTableRequest.getCriteria().getIsPublic() != null) {
//            datasetCriteria.setPublic(datasetTableRequest.getCriteria().getIsPublic());
//        }
//
//        if (!datasetCriteria.isPublic()) {
//            if (datasetCriteria.getCollaborators() == null) {
//                datasetCriteria.setSortCriteria(new ArrayList<>());
//            }
//            datasetCriteria.getCollaborators().add(this.userScope.getUserId());
//        }
//        if (datasetTableRequest.getCriteria().getGrantStatus() != null) {
//            datasetCriteria.setGrantStatus(datasetTableRequest.getCriteria().getGrantStatus());
//        }
//        if (datasetTableRequest.getOrderings() != null) {
//            datasetCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(datasetTableRequest.getOrderings()));
//        }
//        datasetCriteria.setOffset(datasetTableRequest.getOffset());
//        datasetCriteria.setSize(datasetTableRequest.getLength());
//        List<eu.eudat.elastic.entities.Dataset> datasets;
//        try {
//            datasets = datasetRepository.exists() ?
//                    datasetRepository.queryIds(datasetCriteria) : null;
//        } catch (Exception ex) {
//            logger.warn(ex.getMessage(), ex);
//            datasets = null;
//        }

        UserEntity userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(this.userScope.getUserIdSafe()).build();
//        QueryableList<eu.eudat.data.Dataset> items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
        QueryableList<DescriptionEntity> items;
        //TODO
//        if (datasets != null) {
//
//            if (!datasets.isEmpty()) {
//                //items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
//                final List<UUID> datasetIds = datasets.stream().map(datasetE -> UUID.fromString(datasetE.getId())).distinct().collect(Collectors.toList());
//                items = databaseRepository.getDatasetDao().filterFromElastic(datasetTableRequest.getCriteria(), datasetIds).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
//                //items.where((builder, root) -> root.get("id").in(datasetIds));
//            } else {
//                items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
//                //items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
//            }
//        } else {
            items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
//        }
        List<Integer> roles = new LinkedList<>();
        QueryableList<DescriptionEntity> pagedItems;
        QueryableList<DescriptionEntity> authItems;
        if (!datasetTableRequest.getCriteria().getIsPublic()) {
            if (this.userScope.getUserIdSafe() == null) {
                throw new UnauthorisedException("You are not allowed to access those datasets");
            }
            if (datasetTableRequest.getCriteria().getRole() != null) {
                roles.add(datasetTableRequest.getCriteria().getRole());
            }
            authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles).distinct();
            pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
        } else {
            if (this.userScope.getUserIdSafe() != null && datasetTableRequest.getCriteria().getRole() != null) {
                items.where((builder, root) -> {
                    Join userJoin = root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT);
                    return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), this.userScope.getUserId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
                });
            }
            String[] strings = new String[1];
            //strings[0] = "-dmp:publishedAt|join|";
            //datasetTableRequest.getOrderings().setFields(strings);
            authItems = items;
            pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
        }
        DataTableData<DatasetListingModel> dataTable = new DataTableData<>();


        dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList()));

        dataTable.setTotalCount(authItems.count());

        //CompletableFuture.allOf(itemsFuture, countFuture).join();
        return dataTable;
    }

    public DataTableData<DatasetListingModel> getPaged(DatasetPublicTableRequest datasetTableRequest) throws Exception {
        Long count = 0L; 
        //TODO
//        DatasetCriteria datasetCriteria = new DatasetCriteria();
//        datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
//        datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile());
//        datasetCriteria.setDmps(datasetTableRequest.getCriteria().getDmpIds());
//        datasetCriteria.setGrants(datasetTableRequest.getCriteria().getGrants());
//        if (datasetTableRequest.getOrderings() != null) {
//            datasetCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(datasetTableRequest.getOrderings()));
//        }
//        datasetCriteria.setOffset(datasetTableRequest.getOffset());
//        datasetCriteria.setSize(datasetTableRequest.getLength());
//        List<eu.eudat.elastic.entities.Dataset> datasets;
//        try {
//            datasets = datasetRepository.exists() ?
//                    datasetRepository.queryIds(datasetCriteria) : new LinkedList<>();
//            count = datasetRepository.exists() ? datasetRepository.count(datasetCriteria) : 0L;
//        } catch (Exception ex) {
//            logger.warn(ex.getMessage());
//            datasets = null;
//        }
        /*datasetTableRequest.setQuery(databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class)));
        QueryableList<Dataset> items = datasetTableRequest.applyCriteria();*/
        QueryableList<DescriptionEntity> items;
        //TODO
//        if (datasets != null) {
//            if (!datasets.isEmpty()) {
//                items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
//                List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
//                items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
//            } else
//                items = datasetTableRequest.applyCriteria();
//                items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
//        } else {
            items = datasetTableRequest.applyCriteria();
//        }

        if (this.userScope.isSet() && datasetTableRequest.getCriteria().getRole() != null) {
            items.where((builder, root) -> {
                Join userJoin = root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT);
                return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), this.userScope.getUserId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
            });
        }
        List<String> strings = new ArrayList<>();
        strings.add("-dmp:publishedAt|join|");
        datasetTableRequest.getOrderings().setFields(strings);
        if (count == 0L) {
            count = items.count();
        }
        QueryableList<DescriptionEntity> pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
        DataTableData<DatasetListingModel> dataTable = new DataTableData<>();

        List<DatasetListingModel> datasetListis = pagedItems.
                select(this::mapModel);

        dataTable.setData(datasetListis.stream().filter(Objects::nonNull).collect(Collectors.toList()));

        dataTable.setTotalCount(count);
        //CompletableFuture.allOf(countFuture).join();
        return dataTable;
    }

    public DatasetWizardModel getSingle(String id) throws InvalidApplicationException {
        DatasetWizardModel dataset = new DatasetWizardModel();
        DescriptionEntity descriptionEntityEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
        ;
        DmpEntity dmp = databaseRepository.getDmpDao().find(this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntityEntity.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first().getDmpId());
        if (
                //TODO
//                dmp.getUsers()
//                                    .stream().filter(userInfo -> this.userScope.getUserIdSafe().equals(userInfo.getUser().getId()))
//                                    .collect(Collectors.toList()).size() == 0 && 
                        !dmp.getAccessType().equals(DmpAccessType.Public))
            throw new UnauthorisedException();
        dataset.setDatasetProfileDefinition(getPagedProfile(dataset, descriptionEntityEntity));
        dataset.fromDataModel(descriptionEntityEntity);

        // Creates the Criteria to get all version of DescriptionTemplate in question.
        DatasetProfileCriteria profileCriteria = new DatasetProfileCriteria();
        UUID profileId = this.queryFactory.query(DescriptionTemplateQuery.class).ids(descriptionEntityEntity.getDescriptionTemplateId()).first().getGroupId();
        List<UUID> uuidList = new LinkedList<>();
        uuidList.add(profileId);
        profileCriteria.setGroupIds(uuidList);
        profileCriteria.setAllVersions(true);

        List<DescriptionTemplateEntity> profileVersions = databaseRepository.getDatasetProfileDao().getWithCriteria(profileCriteria)
                .orderBy(((builder, root) -> builder.desc(root.get("version"))))
                .toList();
        List<DescriptionTemplateEntity> profileVersionsIncluded = new LinkedList<>();

        // Iterate through the versions and remove those that are not included in the DMP of the dataset in question.
        for (DescriptionTemplateEntity version : profileVersions) {
            for (AssociatedProfile p : dataset.getDmp().getProfiles()) {
                if (version.getId().toString().equals(p.getDescriptionTemplateId().toString())) {
                    profileVersionsIncluded.add(version);
                }
            }
        }

        // Sort the list with the included Versions.
        Stream<DescriptionTemplateEntity> sorted = profileVersionsIncluded.stream().sorted(Comparator.comparing(DescriptionTemplateEntity::getVersion).reversed());

        // Make the Stream into List and get the first item.
        List<DescriptionTemplateEntity> profiles = sorted.collect(Collectors.toList());
        if (profiles.isEmpty())
            throw new NoSuchElementException("No profiles found for the specific Dataset");

        DescriptionTemplateEntity profile = profiles.get(0);

        // Check if the dataset is on the latest Version.
        boolean latestVersion = profile.getVersion().toString().equals(this.queryFactory.query(DescriptionTemplateQuery.class).ids(descriptionEntityEntity.getDescriptionTemplateId()).first().getVersion());
        dataset.setIsProfileLatestVersion(latestVersion);

//        eu.eudat.elastic.entities.Dataset datasetElastic; //TODO
//        try {
//            datasetElastic = datasetRepository.exists() ?
//                    datasetRepository.findDocument(id) : new eu.eudat.elastic.entities.Dataset();
//        } catch (Exception ex) {
//            logger.warn(ex.getMessage());
//            datasetElastic = null;
//        }
//        if (datasetElastic != null && datasetElastic.getTags() != null && !datasetElastic.getTags().isEmpty()) {
//            dataset.setTags(datasetElastic.getTags());
//        }

        /*if (datasetElastic != null && datasetElastic.getLabel() != null && !datasetElastic.getLabel().isEmpty()) {
            dataset.setLabel(datasetElastic.getLabel());
        }*/
        return dataset;
    }

    public DatasetWizardModel getSinglePublic(String id) throws Exception {
        DatasetWizardModel dataset = new DatasetWizardModel();
        DescriptionEntity descriptionEntityEntity = databaseRepository.getDatasetDao().isPublicDataset(UUID.fromString(id));
        DmpEntity dmp = databaseRepository.getDmpDao().find(this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntityEntity.getId()).isActive(IsActive.Active).first().getDmpId());
        if (descriptionEntityEntity != null && descriptionEntityEntity.getStatus() == DescriptionStatus.Finalized && dmp.getStatus().getValue() == 1) {
            dataset.setDatasetProfileDefinition(getPagedProfile(dataset, descriptionEntityEntity));
            dataset.fromDataModel(descriptionEntityEntity);
            return dataset;
        } else {
            throw new Exception("Selected dataset is not public");
        }
    }

    public DatasetOverviewModel getOverviewSingle(String id, boolean isPublic) throws Exception {
        DescriptionEntity descriptionEntityEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id));
        if (descriptionEntityEntity.getIsActive() == IsActive.Inactive) {
            throw new Exception("Dataset is deleted.");
        }
        DmpEntity dmp = databaseRepository.getDmpDao().find(this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntityEntity.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first().getDmpId());
        if (!isPublic && !this.userScope.isSet()) {
            throw new UnauthorisedException();
        } else
        if (!isPublic
        //TODO        
//                && dmp.getUsers()
//                                                 .stream().noneMatch(userInfo -> this.userScope.getUserIdSafe().equals(userInfo.getUser().getId()))
        ) {
            throw new UnauthorisedException();
        } else if (isPublic && !dmp.getAccessType().equals(DmpAccessType.Public)) {
            throw new ForbiddenException("Selected Dataset is not public");
        }
        DatasetOverviewModel dataset = new DatasetOverviewModel();
        dataset.fromDataModel(descriptionEntityEntity);

        return dataset;
    }

    public DescriptionEntity getEntitySingle(UUID id) throws InvalidApplicationException {
        return databaseRepository.getDatasetDao().find(id);
    }

    public PagedDatasetProfile getPagedProfile(DatasetWizardModel dataset, DescriptionEntity descriptionEntityEntity) {
//        eu.eudat.models.data.user.composite.DatasetProfile datasetprofile = userManager.generateDatasetProfileModel(this.queryFactory.query(DescriptionTemplateQuery.class).ids(descriptionEntityEntity.getDescriptionTemplateId()).first());
//        datasetprofile.setStatus(dataset.getStatus().getValue());
//        if (descriptionEntityEntity.getProperties() != null) {
//            JSONObject jObject = new JSONObject(descriptionEntityEntity.getProperties());
//            Map<String, Object> properties = jObject.toMap();
//            datasetprofile.fromJsonObject(properties);
//        }
        PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile();
//        pagedDatasetProfile.buildPagedDatasetProfile(datasetprofile);
        return pagedDatasetProfile;
    }

    private XWPFDocument getWordDocument(ConfigLoader configLoader, DescriptionEntity descriptionEntityEntity, VisibilityRuleService visibilityRuleService) throws IOException, InvalidApplicationException {
        WordBuilder wordBuilder = new WordBuilder(this.environment, configLoader);
        DatasetWizardModel dataset = new DatasetWizardModel();
        XWPFDocument document = configLoader.getDatasetDocument();

        DmpEntity dmpEntity = databaseRepository.getDmpDao().find(this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntityEntity.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first().getDmpId());
        if (!dmpEntity.getAccessType().equals(DmpAccessType.Public)
                //TODO
//                && dmpEntity.getUsers().stream().filter(userInfo -> this.userScope.getUserIdSafe().equals(userInfo.getUser().getId())).collect(Collectors.toList()).size() == 0
        )
            throw new UnauthorisedException();

        wordBuilder.fillFirstPage(dmpEntity, descriptionEntityEntity, document, true);
        wordBuilder.fillFooter(dmpEntity, descriptionEntityEntity, document, true);

        int powered_pos = wordBuilder.findPosOfPoweredBy(document);
        XWPFParagraph powered_par = null;
        XWPFParagraph argos_img_par = null;
        if(powered_pos != -1) {
            powered_par = document.getParagraphArray(powered_pos);
            argos_img_par = document.getParagraphArray(powered_pos + 1);
        }

//        wordBuilder.addParagraphContent(datasetEntity.getLabel(), document, ParagraphStyle.HEADER1, BigInteger.ZERO);

        // Space below Dataset title.
//        XWPFParagraph parBreakDataset = document.createParagraph();
//
//        XWPFParagraph datasetTemplateParagraph = document.createParagraph();
//        datasetTemplateParagraph.setStyle("Heading2");
//        XWPFRun runDatasetTemplate1 = datasetTemplateParagraph.createRun();
//        runDatasetTemplate1.setText("Template: ");
//        runDatasetTemplate1.setBold(true);
//        runDatasetTemplate1.setFontSize(12);
//        XWPFRun runDatasetTemplate = datasetTemplateParagraph.createRun();
//        runDatasetTemplate.setText(datasetEntity.getProfile().getLabel());
//        runDatasetTemplate.setColor("2E75B6");
//        runDatasetTemplate.setBold(true);
//        runDatasetTemplate.setFontSize(12);
//
//        wordBuilder.addParagraphContent(datasetEntity.getDescription(), document, ParagraphStyle.HTML, BigInteger.ZERO);

        /*XWPFParagraph externalReferencesParagraph = document.createParagraph();
        externalReferencesParagraph.setStyle("Heading2");
        XWPFRun externalReferencesRun = externalReferencesParagraph.createRun();
        externalReferencesRun.setText("External References");
        externalReferencesRun.setColor("2E75B6");
        externalReferencesRun.setBold(true);
        externalReferencesRun.setFontSize(12);

        wordBuilder.addParagraphContent("Data Repositories", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.getDatasetDataRepositories().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getDatasetDataRepositories().stream().map(DatasetDataRepository::getDataRepository).map(DataRepository::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.TEXT, BigInteger.ZERO);
        }
        wordBuilder.addParagraphContent("External Datasets", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.getDatasetExternalDatasets().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getDatasetExternalDatasets().stream().map(DatasetExternalDataset::getExternalDataset).map(ExternalDataset::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.TEXT, BigInteger.ZERO);
        }
        wordBuilder.addParagraphContent("Registries", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.getRegistries().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getRegistries().stream().map(Registry::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.TEXT, BigInteger.ZERO);
        }
        wordBuilder.addParagraphContent("Services", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.getServices().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getServices().stream().map(DatasetService::getService).map(Service::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.TEXT, BigInteger.ZERO);
        }*/
        /*wordBuilder.addParagraphContent("Tags", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getServices().stream().map(DatasetService::getService).map(Service::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.HEADER4, BigInteger.ZERO);
        }*/

        Map<String, Object> properties = new HashMap<>();
        if (descriptionEntityEntity.getProperties() != null) {
            JSONObject jObject = new JSONObject(descriptionEntityEntity.getProperties());
            properties = jObject.toMap();
        }

//        wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO);
        PagedDatasetProfile pagedDatasetProfile = getPagedProfile(dataset, descriptionEntityEntity);
        visibilityRuleService.setProperties(properties);
        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
        wordBuilder.build(document, pagedDatasetProfile, visibilityRuleService);
        String label = descriptionEntityEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", "");
//        File exportFile = new File(label + ".docx");

        // Removes the top empty headings.
//        for (int i = 0; i < 6; i++) {
//            document.removeBodyElement(0);
//        }

        if(powered_pos != -1) {
            document.getLastParagraph().setPageBreak(false);
            document.createParagraph();
            document.setParagraph(powered_par, document.getParagraphs().size() - 1);

            document.createParagraph();
            document.setParagraph(argos_img_par, document.getParagraphs().size() - 1);

            document.removeBodyElement(powered_pos + 1);
            document.removeBodyElement(powered_pos + 1);
        }

        return document;
        //FileOutputStream out = new FileOutputStream(exportFile);
       // document.write(out);
       // out.close();
      //  return exportFile;
    }

    private XWPFDocument getLightWordDocument(ConfigLoader configLoader, DatasetWizardModel dataset, VisibilityRuleService visibilityRuleService) throws IOException {
        WordBuilder wordBuilder = new WordBuilder(this.environment, configLoader);
        XWPFDocument document = configLoader.getDocument();

        // Space below Dataset title.
        XWPFParagraph parBreakDataset = document.createParagraph();

        Map<String, Object> properties = new HashMap<>();
        if (dataset.getDatasetProfileDefinition() != null) {
            JSONObject jObject = new JSONObject(propertiesModelToString(dataset.getDatasetProfileDefinition()));
            properties = jObject.toMap();
        }

        wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO, 0);
        visibilityRuleService.setProperties(properties);
        visibilityRuleService.buildVisibilityContext(dataset.getDatasetProfileDefinition().getRules());
        wordBuilder.build(document, dataset.getDatasetProfileDefinition(), visibilityRuleService);

        // Removes the top empty headings.
        for (int i = 0; i < 6; i++) {
            document.removeBodyElement(0);
        }

        return document;
        //FileOutputStream out = new FileOutputStream(exportFile);
        // document.write(out);
        // out.close();
        //  return exportFile;
    }

    public FileEnvelope getWordDocumentFile(ConfigLoader configLoader, String id, VisibilityRuleService visibilityRuleService) throws IOException, InvalidApplicationException {
        DescriptionEntity descriptionEntityEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
        DmpEntity dmp = databaseRepository.getDmpDao().find(this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntityEntity.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first().getDmpId());
        if (!dmp.getAccessType().equals(DmpAccessType.Public)
        //TODO        
//                && dmp.getUsers()
//                                                               .stream().filter(userInfo -> this.userScope.getUserIdSafe().equals(userInfo.getUser().getId()))
//                                                               .collect(Collectors.toList()).size() == 0
        )
            throw new UnauthorisedException();
        String label = descriptionEntityEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", "");
        FileEnvelope exportEnvelope = new FileEnvelope();
        exportEnvelope.setFilename(label + ".docx");
        String uuid = UUID.randomUUID().toString();
        File exportFile = new File(this.environment.getProperty("temp.temp") + uuid + ".docx");
        XWPFDocument document = getWordDocument(configLoader, descriptionEntityEntity, visibilityRuleService);
        FileOutputStream out = new FileOutputStream(exportFile);
         document.write(out);
         out.close();
         exportEnvelope.setFile(exportFile);
          return exportEnvelope;
    }

    public String getWordDocumentText (DescriptionEntity descriptionEntityEntity) throws Exception {
        DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(descriptionEntityEntity);
        datasetWizardModel.setDatasetProfileDefinition(this.getPagedProfile(datasetWizardModel, descriptionEntityEntity));
        XWPFDocument document = getLightWordDocument(this.configLoader, datasetWizardModel, new VisibilityRuleServiceImpl());
        XWPFWordExtractor extractor = new XWPFWordExtractor(document);
        return extractor.getText();/*.replaceAll("\n\\s*", " ");*/
    }

    public FileEnvelope getXmlDocument(String id, VisibilityRuleService visibilityRuleService) throws InstantiationException, IllegalAccessException, IOException, InvalidApplicationException {
        ExportXmlBuilder xmlBuilder = new ExportXmlBuilder();
        DatasetWizardModel dataset = new DatasetWizardModel();
        DescriptionEntity descriptionEntityEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
        DmpEntity dmp = databaseRepository.getDmpDao().find(this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntityEntity.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first().getDmpId());
        if (!dmp.getAccessType().equals(DmpAccessType.Public) 
                //TODO
//                && dmp.getUsers()
//                         .stream().filter(userInfo -> this.userScope.getUserIdSafe().equals(userInfo.getUser().getId()))
//                         .collect(Collectors.toList()).size() == 0
        )
            throw new UnauthorisedException();
        Map<String, Object> properties = new HashMap<>();
        if (descriptionEntityEntity.getProperties() != null) {
            JSONObject jobject = new JSONObject(descriptionEntityEntity.getProperties());
            properties = jobject.toMap();
        }
        PagedDatasetProfile pagedDatasetProfile = getPagedProfile(dataset, descriptionEntityEntity);
        visibilityRuleService.setProperties(properties);
        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
        File file = xmlBuilder.build(pagedDatasetProfile, descriptionEntityEntity.getDescriptionTemplateId(), visibilityRuleService, environment);
        FileEnvelope fileEnvelope = new FileEnvelope();
        fileEnvelope.setFile(file);
        String label = descriptionEntityEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", "");
        fileEnvelope.setFilename(label);
        return fileEnvelope;
    }

    public DescriptionEntity createOrUpdate(DatasetWizardModel datasetWizardModel) throws Exception {
        Boolean sendNotification = false;
        DescriptionEntity tempDescriptionEntity = null;
        DmpEntity dmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(datasetWizardModel.getDmp().getId());
        if (datasetWizardModel.getId() != null) {
           tempDescriptionEntity = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(datasetWizardModel.getId());
            if (tempDescriptionEntity != null) {
            	
            	 Instant dbTime = tempDescriptionEntity.getCreatedAt().truncatedTo(ChronoUnit.SECONDS);
                 Instant modelTime = datasetWizardModel.getModified().truncatedTo(ChronoUnit.SECONDS);
                if (modelTime.toEpochMilli() != dbTime.toEpochMilli()) {
                    throw new Exception("Dataset has been modified already by another user.");
                }
                sendNotification = true;
            }
        } else {
            metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
        }
        if (dmp.getStatus().equals(DmpStatus.Finalized) && datasetWizardModel.getId() != null)
            throw new Exception("DMP is finalized, therefore Dataset cannot be edited.");
        DescriptionEntity descriptionEntity = datasetWizardModel.toDataModel();
//        descriptionEntity.setDmpId(dmp.getId()); //TODO
        descriptionEntity.setProperties(propertiesModelToString(datasetWizardModel.getDatasetProfileDefinition()));
//        if (this.apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().getClient() != null) {
//            this.getTagsFromProfile(datasetWizardModel, descriptionEntity);
//        } //TODO
        if (datasetWizardModel.getStatus() == DescriptionStatus.Finalized) {
            String failedField = checkDatasetValidation(descriptionEntity);
            if (failedField != null) {
                throw new Exception("Field value of " + failedField + " must be filled.");
            }
        }
        UserEntity userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(this.userScope.getUserId()).build();
        descriptionEntity.setDmpDescriptionTemplateId(userInfo.getId());

//        createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), descriptionEntity);//TODO
        createExternalDatasetsIfTheyDontExist(descriptionEntity);
//        createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), descriptionEntity);
        createServicesIfTheyDontExist(descriptionEntity);
        DescriptionEntity descriptionEntity1 = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(descriptionEntity);
        datasetWizardModel.setId(descriptionEntity1.getId());
//        if (datasetWizardModel.getDmp().getGrant() == null) { //TODO
//            DmpEntity dmp1 = databaseRepository.getDmpDao().find(this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntity1.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first().getDmpId());
//            
//            datasetWizardModel.setDmp(new DataManagementPlan().fromDataModelNoDatasets(dmp1));
//        }
//        descriptionEntity1.setDescriptionTemplateId(datasetWizardModel.getProfile().getId()); //TODO
//        datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset1));
        UUID dmpId = this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(descriptionEntity1.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first().getDmpId();
        DmpEntity dmp1 = databaseRepository.getDmpDao().find(dmpId);
//        dmp1.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));//TODO
        //updateTags(descriptionEntity1, datasetWizardModel.getTags()); //TODO
        if (sendNotification) {
            if (descriptionEntity1.getStatus() != DescriptionStatus.Finalized) {
                this.sendNotification(descriptionEntity1, dmp1, userInfo/*, NotificationType.DATASET_MODIFIED*/);
            } else {
                this.sendNotification(descriptionEntity1, dmp1, userInfo/*, NotificationType.DATASET_MODIFIED_FINALISED*/);
            }
        }

        this.deleteOldFilesAndAddNew(datasetWizardModel, userInfo);


        return descriptionEntity1;
    }

    private void deleteOldFilesAndAddNew(DatasetWizardModel datasetWizardModel, UserEntity userInfo) throws JsonProcessingException, InvalidApplicationException {
        // Files in DB for this entityId which are NOT DELETED
        List<FileUpload> fileUploads = fileManager.getCurrentFileUploadsForEntityId(datasetWizardModel.getId());
        List<String> fileUploadIds = fileUploads.stream().map(fileUpload -> fileUpload.getId().toString()).collect(Collectors.toList());

        ObjectMapper mapper = new ObjectMapper();
        mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
        String json = mapper.writeValueAsString(datasetWizardModel.getDatasetProfileDefinition());
        JsonNode propertiesJson = mapper.readTree(json);

        Set<JsonNode> uploadNodes = new HashSet<>();
        uploadNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "upload", true));

        uploadNodes.forEach(node -> {
            JsonNode value = node.get("value");
            if (value != null && !value.toString().equals("\"\"") && !value.toString().equals("null")) {
                String stringValue = value.toString().replaceAll("=", ":");
                JSONObject values = new JSONObject(stringValue);
                Map<String, Object> data = ((JSONObject) values).toMap();

                int index = fileUploadIds.indexOf(data.get("id").toString());
                if(index != -1) {
                    // file in DB is the same as file in the Dataset
                    fileUploadIds.remove(index);
                    fileUploads.remove(index);
                } else {
                    // new file
                    this.fileManager.createFile(data.get("id").toString(), data.get("name").toString(), data.get("type").toString(), datasetWizardModel.getId().toString(), FileUpload.EntityType.DATASET, userInfo);
                }
            }
        });

        // old files in DB that are not contained anymore in the Dataset -> mark them as Deleted
        fileUploads.forEach(fileUpload -> {
            fileManager.markOldFileAsDeleted(fileUpload);
        });
    }

    private void sendNotification(DescriptionEntity descriptionEntity, DmpEntity dmp, UserEntity user/*, NotificationType notificationType*/) throws InvalidApplicationException {
        List<DmpUserEntity> userDMPS = this.queryFactory.query(DmpUserQuery.class).dmpIds(dmp.getId()).collect();
        for (DmpUserEntity userDMP : userDMPS) {
            if (!userDMP.getUserId().equals(user.getId())) {
//                Notification notification = new Notification(); //TODO
//                notification.setUserId(user);
//                notification.setType(notificationType);
//                notification.setNotifyState(NotifyState.PENDING);
//                notification.setIsActive(ActiveStatus.ACTIVE);
//                notification.setData("{" +
//                        "\"userId\": \"" + userDMP.getUserId() + "\"" +
//                        ", \"id\": \"" + descriptionEntity.getId() + "\"" +
//                        ", \"name\": \"" + descriptionEntity.getLabel() + "\"" +
//                        ", \"path\": \"" + notificationPaths.get(notificationType) + "\"" +
//                        "}");
//                notification.setCreatedAt(new Date());
//                notification.setUpdatedAt(notification.getCreatedAt());
//                notification.setContactTypeHint(ContactType.EMAIL);
//
//                UserContactInfoQuery query = this.queryFactory.query(UserContactInfoQuery.class).userIds(userDMP.getUserId());
//                query.setOrder(new Ordering().addAscending(UserContactInfo._ordinal));
//                notification.setContactHint(query.first().getValue());
//                databaseRepository.getNotificationDao().createOrUpdate(notification);
            }
        }

    }

    public String checkDatasetValidation(DescriptionEntity descriptionEntity) throws Exception {
        List<String> datasetProfileValidators = new LinkedList<>();
        DescriptionTemplateEntity profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(descriptionEntity.getDescriptionTemplateId());
        DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder builder = builderFactory.newDocumentBuilder();
        Document xmlDocument = builder.parse(new ByteArrayInputStream(profile.getDefinition().getBytes()));

        XPath xPath = XPathFactory.newInstance().newXPath();
        String expression = "//validation/@type[.=1]/ancestor::field/@id";
        NodeList nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);

        for (int i = 0; i < nodeList.getLength(); i++) {
            Node node = nodeList.item(i);
            datasetProfileValidators.add(node.getNodeValue());
        }

        expression = "//validation/@type[.=1]/ancestor::fieldSet";
        nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);


        JSONObject obj = new JSONObject(descriptionEntity.getProperties());
        VisibilityRuleService visibilityRuleService = new VisibilityRuleServiceImpl();
        visibilityRuleService.setProperties(obj.toMap());

        //descriptionEntity.setDescriptionTemplateId(profile.getId()); //TODO
        PagedDatasetProfile pagedDatasetProfile = this.getPagedProfile(new DatasetWizardModel(), descriptionEntity);
        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());


        String failedField = null;

        for (String validator : datasetProfileValidators) {
            if (obj.has(validator) && isNullOrEmpty(obj.getString(validator)) && isElementVisible(nodeList, validator, visibilityRuleService)) {
                //throw new Exception("Field value of " + validator + " must be filled.");
                failedField = validator;
                break;
            }
        }

        return failedField;
    }

    private boolean isNullOrEmpty(String value) {
        return value == null || value.trim().isEmpty();
    }

    private boolean isElementVisible(NodeList nodeList, String id, VisibilityRuleService visibilityRuleService) {
        Element fieldSet = null;
        for (int i = 0; i < nodeList.getLength(); i++) {
            Node node = nodeList.item(i);
            for (int j = 0; j < node.getChildNodes().getLength(); j++) {
                Node fcnode = node.getChildNodes().item(j);
                if (fcnode.getNodeName().equals("fields")) {
                    for(int k = 0; k < fcnode.getChildNodes().getLength(); k++) {
                        Node scnode = fcnode.getChildNodes().item(k);
                        if (scnode.getNodeName().equals("field") && scnode.getAttributes().getNamedItem("id").getNodeValue().equals(id)) {
                            fieldSet = (Element) node;
                        }
                    }
                }
            }
        }
        if (fieldSet != null) {
            return visibilityRuleService.isElementVisible(id) && visibilityRuleService.isElementVisible(fieldSet.getAttribute("id"));
        } else {
            return visibilityRuleService.isElementVisible(id);
        }
    }

    private String propertiesModelToString(PagedDatasetProfile pagedDatasetProfile) {
        Map<String, Object> values = new LinkedHashMap<>();
        pagedDatasetProfile.toMap(values);
        JSONObject jobject = new JSONObject(values);
        return jobject.toString();
    }

    public void updateTags(DescriptionEntity descriptionEntityEntity, List<TagEntity> tags) throws Exception {
       // if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) {
        /*eu.eudat.elastic.entities.Dataset dataset = new eu.eudat.elastic.entities.Dataset();
        dataset.setId(datasetWizardModel.getId().toString());
        if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) {
            DatasetCriteria criteria = new DatasetCriteria();
            criteria.setTags(datasetWizardModel.getTags());
            List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
                                       .filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> datasetWizardModel.getTags().stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
            if (tags.isEmpty()) {
                datasetWizardModel.getTags().forEach(tag -> tag.setId(UUID.randomUUID().toString()));
                dataset.setTags(datasetWizardModel.getTags());
            } else {
                dataset.setTags(tags);
            }
        }
        dataset.setLabel(datasetWizardModel.getLabel());
        dataset.setDescription(datasetWizardModel.getDescription());
        dataset.setTemplate(datasetWizardModel.getProfile());
        dataset.setStatus(datasetWizardModel.getStatus());
        dataset.setDmp(datasetWizardModel.getDmp().getId());
        dataset.setGroup(datasetWizardModel.getDmp().getGroupId());
        dataset.setGrant(datasetWizardModel.getDmp().getGrant().getId());
        if (datasetWizardModel.getDmp().getUsers() != null) {
            dataset.setCollaborators(datasetWizardModel.getDmp().getUsers().stream().map(user -> {
                Collaborator collaborator = new Collaborator();
                collaborator.setId(user.getId().toString());
                collaborator.setName(user.getName());
                return collaborator;
            }).collect(Collectors.toList()));
        }
        DataManagementPlanCriteria dmpCriteria = new DataManagementPlanCriteria();
        dmpCriteria.setAllVersions(true);
        dmpCriteria.setGroupIds(Collections.singletonList(datasetWizardModel.getDmp().getGroupId()));
        apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream()
                  .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> dataset.setLastVersion(dmp.getId().equals(datasetWizardModel.getDmp().getId())));
        apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream().filter(DMP::isPublic)
                  .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> dataset.setLastPublicVersion(dmp.getId().equals(datasetWizardModel.getDmp().getId())));
        if (dataset.getLastVersion() == null) {
            dataset.setLastVersion(true);
        }
        if (dataset.getLastPublicVersion() == null) {
            dataset.setLastPublicVersion(false);
        }
        if (datasetWizardModel.getDmp().getOrganisations() != null) {
            dataset.setOrganizations(datasetWizardModel.getDmp().getOrganisations().stream().map(org -> {
                Organization organization = new Organization();
                organization.setId(org.getId());
                organization.setName(org.getName());
                return organization;
            }).collect(Collectors.toList()));
        }
        dataset.setPublic(datasetWizardModel.getDmp().getPublic());
        dataset.setGrantStatus(datasetWizardModel.getDmp().getGrant().getStatus());
        dataset.setFormData(this.getWordDocumentText(datasetWizardModel));*/
//        DatasetMapper mapper = new DatasetMapper(apiContext, this); //TODO
//        eu.eudat.elastic.entities.Dataset dataset = mapper.toElastic(descriptionEntityEntity, tags);
//        apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(dataset);
       // }
    }


    //TODO implement it on create
//    private void createRegistriesIfTheyDontExist(RegistryDao registryDao, DescriptionEntity descriptionEntity) throws InvalidApplicationException {
//        
//        if (descriptionEntity.getRegistries() != null && !descriptionEntity.getRegistries().isEmpty()) {
//            for (Registry registry : descriptionEntity.getRegistries()) {
//                RegistryCriteria criteria = new RegistryCriteria();
//                criteria.setLike(registry.getReference());
//                List<Registry> entries = registryDao.getWithCriteria(criteria).toList();
//                if (entries != null && !entries.isEmpty()) registry.setId(entries.get(0).getId());
//                else {
//                    registry.setCreated(new Date());
//                    registryDao.createOrUpdate(registry);
//                }
//            }
//        }
//    }

    //TODO implement it on create
//    private void createDataRepositoriesIfTheyDontExist(DataRepositoryDao dataRepositoryDao, DescriptionEntity descriptionEntity) throws InvalidApplicationException {
//       
//        if (descriptionEntity.getDatasetDataRepositories() != null && !descriptionEntity.getDatasetDataRepositories().isEmpty()) {
//            for (DatasetDataRepository datasetDataRepository : descriptionEntity.getDatasetDataRepositories()) {
//                DataRepositoryCriteria criteria = new DataRepositoryCriteria();
//                criteria.setLike(datasetDataRepository.getDataRepository().getReference());
//                List<DataRepository> entries = dataRepositoryDao.getWithCriteria(criteria).toList();
//                if (entries != null && !entries.isEmpty()) {
//                    datasetDataRepository.getDataRepository().setId(entries.get(0).getId());
//                    datasetDataRepository.setDataset(descriptionEntity);
//                    descriptionEntity.getDatasetDataRepositories().add(datasetDataRepository);
//                } else {
//                    datasetDataRepository.getDataRepository().setId(UUID.randomUUID());
//                    DataRepository dataRepository = dataRepositoryDao.createOrUpdate(datasetDataRepository.getDataRepository());
//                    datasetDataRepository.setDataset(descriptionEntity);
//                    datasetDataRepository.setDataRepository(dataRepository);
//                    descriptionEntity.getDatasetDataRepositories().add(datasetDataRepository);
//                }
//            }
//        }
//    }

    private void createServicesIfTheyDontExist(DescriptionEntity descriptionEntity) throws InvalidApplicationException {
        //TODO implement it on create
//        if (descriptionEntity.getServices() != null && !descriptionEntity.getServices().isEmpty()) {
//            for (DatasetService service : descriptionEntity.getServices()) {
//                ServiceCriteria criteria = new ServiceCriteria();
//                criteria.setLike(service.getService().getReference());
//                List<Service> entries = databaseRepository.getServiceDao().getWithCriteria(criteria).toList();
//                if (entries != null && !entries.isEmpty()) {
//                    service.setDataset(descriptionEntity);
//                    service.getService().setCreated(new Date());
//                    service.setService(service.getService());
//                    this.databaseRepository.getServiceDao().createOrUpdate(service.getService());
//                    descriptionEntity.getServices().add(service);
//                }
//            }
//        }
    }

    private void createExternalDatasetsIfTheyDontExist(DescriptionEntity descriptionEntity) throws InvalidApplicationException {
        //TODO implement it on create
//        if (descriptionEntity.getDatasetExternalDatasets() != null && !descriptionEntity.getDatasetExternalDatasets().isEmpty()) {
//            for (DatasetExternalDataset datasetExternalDataset : descriptionEntity.getDatasetExternalDatasets()) {
//                ExternalDatasetCriteria criteria = new ExternalDatasetCriteria();
//                criteria.setLike(datasetExternalDataset.getExternalDataset().getReference());
//                List<ExternalDataset> entries = databaseRepository.getExternalDatasetDao().getWithCriteria(criteria).toList();
//                if (entries != null && !entries.isEmpty()) {
//                    datasetExternalDataset.getExternalDataset().setId(entries.get(0).getId());
//                    datasetExternalDataset.setDataset(descriptionEntity);
//                    descriptionEntity.getDatasetExternalDatasets().add(datasetExternalDataset);
//                } else {
//                    datasetExternalDataset.getExternalDataset().setId(UUID.randomUUID());
//                    datasetExternalDataset.setDataset(descriptionEntity);
//                    ExternalDataset externalDataset = databaseRepository.getExternalDatasetDao().createOrUpdate(datasetExternalDataset.getExternalDataset());
//                    datasetExternalDataset.setExternalDataset(externalDataset);
//                    descriptionEntity.getDatasetExternalDatasets().add(datasetExternalDataset);
//                }
//            }
//        }
    }

    public void makePublic(DatasetDao datasetDao, UUID id) throws Exception {
        DescriptionEntity descriptionEntity = datasetDao.find(id);
        if (descriptionEntity.getStatus() != DescriptionStatus.Finalized)
            throw new Exception("You cannot make public a Dataset That Has not Been Finalised");
        datasetDao.createOrUpdate(descriptionEntity);
        metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
    }

    public ResponseEntity<byte[]> getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType) throws IllegalAccessException, IOException, InstantiationException, InvalidApplicationException {
        FileEnvelope envelope = getXmlDocument(id, visibilityRuleService);
        InputStream resource = new FileInputStream(envelope.getFile());
        logger.info("Mime Type of " + envelope.getFilename() + " is " +
                new MimetypesFileTypeMap().getContentType(envelope.getFile()));
        HttpHeaders responseHeaders = new HttpHeaders();
        responseHeaders.setContentLength(envelope.getFile().length());
        responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
        String fileName = envelope.getFilename().replace(" ", "_").replace(",", "_");
        responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName + ".xml");
        responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
        responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");

        byte[] content = org.apache.poi.util.IOUtils.toByteArray(resource);
        resource.close();
        Files.deleteIfExists(envelope.getFile().toPath());

        return new ResponseEntity<>(content,
                responseHeaders,
                HttpStatus.OK);
    }

    public DescriptionEntity createDatasetFromXml(MultipartFile importFile, String dmpId, String datasetProfileId) throws JAXBException, IOException, InvalidApplicationException {
        DatasetImportPagedDatasetProfile importModel = new DatasetImportPagedDatasetProfile();
        JAXBContext jaxbContext;

        // Parses XML into DatasetImport Model.
        try {
            InputStream in = importFile.getInputStream();
            jaxbContext = JAXBContext.newInstance(DatasetImportPagedDatasetProfile.class);
            Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
            DatasetImportPagedDatasetProfile datasetImport = (DatasetImportPagedDatasetProfile) jaxbUnmarshaller.unmarshal(in);
            importModel = datasetImport;
        } catch (IOException e) {
            logger.error(e.getMessage(), e);
        }

        // Checks if XML datasetProfileId GroupId matches the one selected.
        try {
            DescriptionTemplateEntity importDescriptionTemplateEntity = databaseRepository.getDatasetProfileDao().find(UUID.fromString(importModel.getDatasetProfileId()));
            DescriptionTemplateEntity latestVersionDescriptionTemplateEntity = databaseRepository.getDatasetProfileDao().find(UUID.fromString(datasetProfileId));
            if (latestVersionDescriptionTemplateEntity.getGroupId() != importDescriptionTemplateEntity.getGroupId()) {
                throw new Exception();
            }
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
            return null;
        }

        // Creates the Hash Map to place the values of the data set.
        Map<String, String> importMap = importModel.getPages().stream()
                .flatMap(s -> s.getSections().getSection().stream()
                        .flatMap(cFields -> cFields.getCompositeFields().stream()
                                .flatMap(cField -> cField.getCompositeField().stream()
                                        .filter(Objects::nonNull)
                                        .flatMap(fields -> fields.getFields().stream()
                                                .flatMap(field -> field.getField().stream()
                                                        .filter(f -> f.getValue() != null)
                                                )))))
                .collect(Collectors.toMap(DatasetImportField::getId, DatasetImportField::getValue));

        // Transforms map into json file.
        JSONObject jsonDatasetProperties = new JSONObject(importMap);

        // Creates the entity data set to save.
        DescriptionEntity entity = new DescriptionEntity();
        entity.setProperties(jsonDatasetProperties.toString());
        entity.setLabel(importFile.getOriginalFilename());
        DmpEntity dmp = new DmpEntity();
        dmp.setId(UUID.fromString(dmpId));
//        entity.setDmpId(dmp.getId()); //TODO
        entity.setStatus(DescriptionStatus.Draft);
        entity.setCreatedAt(Instant.now());
        entity.setUpdatedAt(Instant.now());
        DescriptionTemplateEntity profile = new DescriptionTemplateEntity();
        profile.setId(UUID.fromString(datasetProfileId));
//        entity.setDescriptionTemplateId(profile.getId());//TODO

        UserEntity userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(this.userScope.getUserId()).build();
        entity.setDmpDescriptionTemplateId(userInfo.getId());

        updateTagsXmlImportDataset(entity);
//        createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), entity);
//        createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), entity);//TODO
        createServicesIfTheyDontExist(entity);
        createExternalDatasetsIfTheyDontExist(entity);

        metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity);
    }

    public void updateTagsXmlImportDataset(DescriptionEntity descriptionEntity) throws IOException {
        // TODO: When tags functionality return.
    }


//    public PagedDatasetProfile getLatestDatasetProfile(DescriptionEntity descriptionEntityEntity, DescriptionTemplateEntity profile) {
//        eu.eudat.models.data.user.composite.DatasetProfile datasetprofile = userManager.generateDatasetProfileModel(profile);
//        datasetprofile.setStatus(descriptionEntityEntity.getStatus().getValue());
//        if (descriptionEntityEntity.getProperties() != null) {
//            JSONObject jobject = new JSONObject(descriptionEntityEntity.getProperties());
//            Map<String, Object> properties = jobject.toMap();
//            datasetprofile.fromJsonObject(properties);
//        }
//        PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile();
//        pagedDatasetProfile.buildPagedDatasetProfile(datasetprofile);
//        return pagedDatasetProfile;
//    }

    public DataTableData<DatasetProfileListingModel> getDatasetProfilesUsedByDatasets(DatasetProfileTableRequestItem datasetProfileTableRequestItem) throws InvalidApplicationException {
        datasetProfileTableRequestItem.getCriteria().setFilter(DatasetProfileCriteria.DatasetProfileFilter.Datasets.getValue());
        datasetProfileTableRequestItem.getCriteria().setUserId(this.userScope.getUserId());

        QueryableList<DescriptionTemplateEntity> items = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(datasetProfileTableRequestItem.getCriteria());
        List<DatasetProfileListingModel> listingModels = items.select(item -> new DatasetProfileListingModel().fromDataModel(item));

        DataTableData<DatasetProfileListingModel> data = new DataTableData<>();
        data.setData(listingModels);
        data.setTotalCount((long) listingModels.size());

        return data;
    }

    public void generateIndex() throws InvalidApplicationException {
        if (this.authorizationService.authorize(Permission.AdminRole)) {
            this.apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList();
            List<DescriptionEntity> descriptionEntityEntities = new ArrayList<>(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().asQueryable().toList());
            descriptionEntityEntities.forEach(datasetEntity -> {
                try {
                    UUID dmpId = this.queryFactory.query(DmpDescriptionTemplateQuery.class).ids(datasetEntity.getDmpDescriptionTemplateId()).isActive(IsActive.Active).first().getDmpId();
                    DmpEntity dmp = databaseRepository.getDmpDao().find(dmpId);
                    //dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList())); //TODO
//                    eu.eudat.elastic.entities.Dataset dataset = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(datasetEntity.getId().toString()); //TODO
//                    updateTags(datasetEntity, dataset != null ? dataset.getTags() : null);
                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                }
            });
        }
    }
//TODO
//    public void getTagsFromProfile(DatasetWizardModel wizardModel, DescriptionEntity descriptionEntity) throws IOException, InvalidApplicationException {
////        descriptionEntity.setDescriptionTemplateId(descriptionEntity.getDescriptionTemplateId()); //TODO
//        wizardModel.setDatasetProfileDefinition(this.getPagedProfile(wizardModel, descriptionEntity));
//        ObjectMapper mapper = new ObjectMapper();
//        String json = mapper.writeValueAsString(wizardModel.getDatasetProfileDefinition());
//        JsonNode propertiesJson = mapper.readTree(json);
//        DatasetCriteria criteria = new DatasetCriteria();
//        criteria.setHasTags(true);
//        List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
//        Set<JsonNode> tagNodes = new HashSet<>();
//        tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "tags", true));
//        tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "schematics", "rda.dataset.keyword"));
//        if(wizardModel.getTags() == null){
//            wizardModel.setTags(new ArrayList<>());
//        }
//        if (!tagNodes.isEmpty()) {
//            tagNodes.forEach(node -> {
//                JsonNode value = node.get("value");
//                if (!value.toString().equals("\"\"") && !value.toString().equals("null")) {
//                    if (value.toString().startsWith("[")) {
//                        String stringValue = value.toString().replaceAll("=", ":");
//                        JSONArray values = new JSONArray(stringValue);
//                        values.iterator().forEachRemaining(element -> {
//                            Map<String, Object> data = ((JSONObject) element).toMap();
//                            this.addTag(tags, wizardModel.getTags(), data.get("id").toString(), data.get("name").toString());
//                        });
//                    } else {
//                        List<String> values = Arrays.asList(value.textValue().split(", "));
//                        List<Tag> tagValues = values.stream().map(stringValue -> new Tag(stringValue, stringValue)).collect(Collectors.toList());
//                        tagValues.iterator().forEachRemaining(tag -> {
//                            this.addTag(tags, wizardModel.getTags(), tag.getId(), tag.getName());
//                        });
//                    }
//                }
//            });
//        }
//    }
//
//    private void addTag(List<Tag> srcTags, List<Tag> dstTags, String id, String name) {
//        Tag tag = new Tag();
//        if(srcTags.stream().anyMatch(intag -> intag.getName().equals(name))) {
//            tag = srcTags.stream().filter(intag -> intag.getName().equals(name)).findFirst().get();
//        } else {
//            tag.setName(name);
//            tag.setId(id);
//        }
//        if (dstTags.stream().noneMatch(intag -> intag.getName().equals(name))) {
//            dstTags.add(tag);
//        }
//    }

    @Transactional
    private DatasetListingModel mapModel(DescriptionEntity item) {
        /*if (item.getProfile() == null)
            return null;*/
        DatasetListingModel listingModel = new DatasetListingModel().fromDataModel(item);
        /*DatasetProfileCriteria criteria = new DatasetProfileCriteria();
        criteria.setGroupIds(Collections.singletonList(item.getProfile().getGroupId()));
        List<DescriptionTemplate> profiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
        boolean islast = false;
        if (!profiles.isEmpty()) {
            profiles = profiles.stream().sorted(Comparator.comparing(DescriptionTemplate::getVersion)).collect(Collectors.toList());
            islast = profiles.get(0).getId().equals(item.getProfile().getId());
        }
        listingModel.setProfileLatestVersion(islast);*/
        return listingModel;
    }
}