package eu.eudat.logic.managers;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.data.dao.criteria.*;
import eu.eudat.data.dao.entities.DataRepositoryDao;
import eu.eudat.data.dao.entities.DatasetDao;
import eu.eudat.data.dao.entities.RegistryDao;
import eu.eudat.data.entities.*;
import eu.eudat.data.enumeration.notification.ActiveStatus;
import eu.eudat.data.enumeration.notification.ContactType;
import eu.eudat.data.enumeration.notification.NotificationType;
import eu.eudat.data.enumeration.notification.NotifyState;
import eu.eudat.data.query.items.table.dataset.DatasetPublicTableRequest;
import eu.eudat.data.query.items.table.dataset.DatasetTableRequest;
import eu.eudat.data.query.items.table.datasetprofile.DatasetProfileTableRequestItem;
import eu.eudat.elastic.criteria.DatasetCriteria;
import eu.eudat.elastic.entities.Tag;
import eu.eudat.elastic.repository.DatasetRepository;
import eu.eudat.exceptions.security.ForbiddenException;
import eu.eudat.exceptions.security.UnauthorisedException;
import eu.eudat.logic.builders.BuilderFactory;
import eu.eudat.logic.builders.entity.UserInfoBuilder;
import eu.eudat.logic.mapper.elastic.DatasetMapper;
import eu.eudat.logic.mapper.elastic.criteria.DmpCriteriaMapper;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.forms.VisibilityRuleService;
import eu.eudat.logic.services.forms.VisibilityRuleServiceImpl;
import eu.eudat.logic.services.operations.DatabaseRepository;
import eu.eudat.logic.utilities.documents.helpers.FileEnvelope;
import eu.eudat.logic.utilities.documents.types.ParagraphStyle;
import eu.eudat.logic.utilities.documents.word.WordBuilder;
import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder;
import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
import eu.eudat.logic.utilities.json.JsonSearcher;
import eu.eudat.models.HintedModelFactory;
import eu.eudat.models.data.dataset.DatasetOverviewModel;
import eu.eudat.models.data.datasetImport.DatasetImportField;
import eu.eudat.models.data.datasetImport.DatasetImportPagedDatasetProfile;
import eu.eudat.models.data.datasetprofile.DatasetProfileListingModel;
import eu.eudat.models.data.datasetprofile.DatasetProfileOverviewModel;
import eu.eudat.models.data.datasetwizard.DatasetWizardModel;
import eu.eudat.models.data.dmp.AssociatedProfile;
import eu.eudat.models.data.dmp.DataManagementPlan;
import eu.eudat.models.data.helpers.common.DataTableData;
import eu.eudat.models.data.listingmodels.DatasetListingModel;
import eu.eudat.models.data.security.Principal;
import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import eu.eudat.queryable.QueryableList;
import eu.eudat.types.Authorities;
import eu.eudat.types.MetricNames;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;
import org.json.JSONArray;
import org.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.web.multipart.MultipartFile;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import javax.activation.MimetypesFileTypeMap;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.transaction.Transactional;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;
import java.io.*;
import java.math.BigInteger;
import java.nio.file.Files;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;

@Component
public class DatasetManager {
    private static final Logger logger = LoggerFactory.getLogger(DatasetManager.class);

    private final Map<NotificationType, String> notificationPaths = Stream.of(new Object[][] {
            {NotificationType.DATASET_MODIFIED, "/datasets/edit"},
            {NotificationType.DATASET_MODIFIED_FINALISED, "/datasets/edit"}
    }).collect(Collectors.toMap(data -> (NotificationType) data[0], data -> (String) data[1]));

    private ApiContext apiContext;
    private DatabaseRepository databaseRepository;
    private DatasetRepository datasetRepository;
    private BuilderFactory builderFactory;
    private UserManager userManager;
    private ConfigLoader configLoader;
    private Environment environment;
    private final MetricsManager metricsManager;
    private final FileManager fileManager;

    @Autowired
    public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment, MetricsManager metricsManager,
                          FileManager fileManager) {
        this.apiContext = apiContext;
        this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
        this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository();
        this.builderFactory = apiContext.getOperationsContext().getBuilderFactory();
        this.userManager = userManager;
        this.configLoader = configLoader;
        this.environment = environment;
        this.metricsManager = metricsManager;
        this.fileManager = fileManager;
    }

    public DataTableData<DatasetListingModel> getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception {
        DatasetCriteria datasetCriteria = new DatasetCriteria();
        datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
        datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetTemplates());
        if (datasetTableRequest.getCriteria().getStatus() != null) {
            datasetCriteria.setStatus(datasetTableRequest.getCriteria().getStatus().shortValue());
        }
        datasetCriteria.setDmps(datasetTableRequest.getCriteria().getDmpIds());
        datasetCriteria.setGroupIds(datasetTableRequest.getCriteria().getGroupIds());
        datasetCriteria.setGrants(datasetTableRequest.getCriteria().getGrants());
        datasetCriteria.setCollaborators(datasetTableRequest.getCriteria().getCollaborators());
        datasetCriteria.setAllowAllVersions(datasetTableRequest.getCriteria().getAllVersions());
        datasetCriteria.setOrganiztions(datasetTableRequest.getCriteria().getOrganisations());
        datasetCriteria.setTags(datasetTableRequest.getCriteria().getTags());
        if (datasetTableRequest.getCriteria().getIsPublic() != null) {
            datasetCriteria.setPublic(datasetTableRequest.getCriteria().getIsPublic());
        }

        if (!datasetCriteria.isPublic()) {
            if (datasetCriteria.getCollaborators() == null) {
                datasetCriteria.setSortCriteria(new ArrayList<>());
            }
            datasetCriteria.getCollaborators().add(principal.getId());
        }
        if (datasetTableRequest.getCriteria().getGrantStatus() != null) {
            datasetCriteria.setGrantStatus(datasetTableRequest.getCriteria().getGrantStatus());
        }
        if (datasetTableRequest.getOrderings() != null) {
            datasetCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(datasetTableRequest.getOrderings()));
        }
        datasetCriteria.setOffset(datasetTableRequest.getOffset());
        datasetCriteria.setSize(datasetTableRequest.getLength());
        List<eu.eudat.elastic.entities.Dataset> datasets;
        try {
            datasets = datasetRepository.exists() ?
                    datasetRepository.queryIds(datasetCriteria) : null;
        } catch (Exception ex) {
            logger.warn(ex.getMessage(), ex);
            datasets = null;
        }

        UserInfo userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
//        QueryableList<eu.eudat.data.entities.Dataset> items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
        QueryableList<eu.eudat.data.entities.Dataset> items;
        if (datasets != null) {

            if (!datasets.isEmpty()) {
                //items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
                final List<UUID> datasetIds = datasets.stream().map(datasetE -> UUID.fromString(datasetE.getId())).distinct().collect(Collectors.toList());
                items = databaseRepository.getDatasetDao().filterFromElastic(datasetTableRequest.getCriteria(), datasetIds).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
                //items.where((builder, root) -> root.get("id").in(datasetIds));
            } else {
                items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
                //items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
            }
        } else {
            items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));//.withFields(Collections.singletonList("id"));
        }
        List<Integer> roles = new LinkedList<>();
        QueryableList<eu.eudat.data.entities.Dataset> pagedItems;
        QueryableList<eu.eudat.data.entities.Dataset> authItems;
        if (!datasetTableRequest.getCriteria().getIsPublic()) {
            if (principal.getId() == null) {
                throw new UnauthorisedException("You are not allowed to access those datasets");
            }
            if (datasetTableRequest.getCriteria().getRole() != null) {
                roles.add(datasetTableRequest.getCriteria().getRole());
            }
            authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles).distinct();
            pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest);
        } else {
            if (principal.getId() != null && datasetTableRequest.getCriteria().getRole() != null) {
                items.where((builder, root) -> {
                    Join userJoin = root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT);
                    return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
                });
            }
            String[] strings = new String[1];
            //strings[0] = "-dmp:publishedAt|join|";
            //datasetTableRequest.getOrderings().setFields(strings);
            authItems = items;
            pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
        }
        DataTableData<DatasetListingModel> dataTable = new DataTableData<>();


        dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList()));

        dataTable.setTotalCount(authItems.count());

        //CompletableFuture.allOf(itemsFuture, countFuture).join();
        return dataTable;
    }

    public DataTableData<DatasetListingModel> getPaged(DatasetPublicTableRequest datasetTableRequest, Principal principal) throws Exception {
        Long count = 0L;
        DatasetCriteria datasetCriteria = new DatasetCriteria();
        datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike());
        datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile());
        datasetCriteria.setDmps(datasetTableRequest.getCriteria().getDmpIds());
        datasetCriteria.setGrants(datasetTableRequest.getCriteria().getGrants());
        if (datasetTableRequest.getOrderings() != null) {
            datasetCriteria.setSortCriteria(DmpCriteriaMapper.toElasticSorting(datasetTableRequest.getOrderings()));
        }
        datasetCriteria.setOffset(datasetTableRequest.getOffset());
        datasetCriteria.setSize(datasetTableRequest.getLength());
        List<eu.eudat.elastic.entities.Dataset> datasets;
        try {
            datasets = datasetRepository.exists() ?
                    datasetRepository.queryIds(datasetCriteria) : new LinkedList<>();
            count = datasetRepository.exists() ? datasetRepository.count(datasetCriteria) : 0L;
        } catch (Exception ex) {
            logger.warn(ex.getMessage());
            datasets = null;
        }
        /*datasetTableRequest.setQuery(databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class)));
        QueryableList<Dataset> items = datasetTableRequest.applyCriteria();*/
        QueryableList<Dataset> items;
        if (datasets != null) {
            if (!datasets.isEmpty()) {
                items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class));
                List<eu.eudat.elastic.entities.Dataset> finalDatasets = datasets;
                items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList())));
            } else
                items = datasetTableRequest.applyCriteria();
                items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()}));
        } else {
            items = datasetTableRequest.applyCriteria();
        }

        if (principal.getId() != null && datasetTableRequest.getCriteria().getRole() != null) {
            items.where((builder, root) -> {
                Join userJoin = root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT);
                return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole()));
            });
        }
        List<String> strings = new ArrayList<>();
        strings.add("-dmp:publishedAt|join|");
        datasetTableRequest.getOrderings().setFields(strings);
        if (count == 0L) {
            count = items.count();
        }
        QueryableList<eu.eudat.data.entities.Dataset> pagedItems = PaginationManager.applyPaging(items, datasetTableRequest);
        DataTableData<DatasetListingModel> dataTable = new DataTableData<>();

        List<DatasetListingModel> datasetListis = pagedItems.
                select(this::mapModel);

        dataTable.setData(datasetListis.stream().filter(Objects::nonNull).collect(Collectors.toList()));

        dataTable.setTotalCount(count);
        //CompletableFuture.allOf(countFuture).join();
        return dataTable;
    }

    public DatasetWizardModel getSingle(String id, Principal principal) {
        DatasetWizardModel dataset = new DatasetWizardModel();
        eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
        if (datasetEntity.getDmp().getUsers()
                                    .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId())
                                    .collect(Collectors.toList()).size() == 0 && !datasetEntity.getDmp().isPublic())
            throw new UnauthorisedException();
        eu.eudat.elastic.entities.Dataset datasetElastic;
        try {
            datasetElastic = datasetRepository.exists() ?
                    datasetRepository.findDocument(id) : new eu.eudat.elastic.entities.Dataset();
        } catch (Exception ex) {
            logger.warn(ex.getMessage());
            datasetElastic = null;
        }
        dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity));
        dataset.fromDataModel(datasetEntity);

        // Creates the Criteria to get all version of DatasetProfile in question.
        DatasetProfileCriteria profileCriteria = new DatasetProfileCriteria();
        UUID profileId = datasetEntity.getProfile().getGroupId();
        List<UUID> uuidList = new LinkedList<>();
        uuidList.add(profileId);
        profileCriteria.setGroupIds(uuidList);
        profileCriteria.setAllVersions(true);

        List<eu.eudat.data.entities.DatasetProfile> profileVersions = databaseRepository.getDatasetProfileDao().getWithCriteria(profileCriteria)
                .orderBy(((builder, root) -> builder.desc(root.get("version"))))
                .toList();
        List<DatasetProfile> profileVersionsIncluded = new LinkedList<>();

        // Iterate through the versions and remove those that are not included in the DMP of the dataset in question.
        for (DatasetProfile version : profileVersions) {
            for (AssociatedProfile p : dataset.getDmp().getProfiles()) {
                if (version.getId().toString().equals(p.getId().toString())) {
                    profileVersionsIncluded.add(version);
                }
            }
        }

        // Sort the list with the included Versions.
        Stream<DatasetProfile> sorted = profileVersionsIncluded.stream().sorted(Comparator.comparing(DatasetProfile::getVersion).reversed());

        // Make the Stream into List and get the first item.
        List<DatasetProfile> profiles = sorted.collect(Collectors.toList());
        if (profiles.isEmpty())
            throw new NoSuchElementException("No profiles found for the specific Dataset");

        DatasetProfile profile = profiles.get(0);

        // Check if the dataset is on the latest Version.
        boolean latestVersion = profile.getVersion().toString().equals(datasetEntity.getProfile().getVersion().toString());
        dataset.setIsProfileLatestVersion(latestVersion);

        if (datasetElastic != null && datasetElastic.getTags() != null && !datasetElastic.getTags().isEmpty()) {
            dataset.setTags(datasetElastic.getTags());
        }

        /*if (datasetElastic != null && datasetElastic.getLabel() != null && !datasetElastic.getLabel().isEmpty()) {
            dataset.setLabel(datasetElastic.getLabel());
        }*/
        return dataset;
    }

    public DatasetWizardModel getSinglePublic(String id) throws Exception {
        DatasetWizardModel dataset = new DatasetWizardModel();
        eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().isPublicDataset(UUID.fromString(id));

        if (datasetEntity != null && datasetEntity.getStatus() == 1 && datasetEntity.getDmp().getStatus() == 1) {
            dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity));
            dataset.fromDataModel(datasetEntity);
            return dataset;
        } else {
            throw new Exception("Selected dataset is not public");
        }
    }

    public DatasetOverviewModel getOverviewSingle(String id, Principal principal, boolean isPublic) throws Exception {
        Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id));
        if (datasetEntity.getStatus() == Dataset.Status.DELETED.getValue()) {
            throw new Exception("Dataset is deleted.");
        }
        if (!isPublic && principal == null) {
            throw new UnauthorisedException();
        } else
        if (!isPublic && datasetEntity.getDmp().getUsers()
                                                 .stream().noneMatch(userInfo -> userInfo.getUser().getId() == principal.getId())) {
            throw new UnauthorisedException();
        } else if (isPublic && !datasetEntity.getDmp().isPublic()) {
            throw new ForbiddenException("Selected Dataset is not public");
        }
        DatasetOverviewModel dataset = new DatasetOverviewModel();
        dataset.fromDataModel(datasetEntity);

        return dataset;
    }

    public Dataset getEntitySingle(UUID id) {
        return databaseRepository.getDatasetDao().find(id);
    }

    public PagedDatasetProfile getPagedProfile(DatasetWizardModel dataset, eu.eudat.data.entities.Dataset datasetEntity) {
        eu.eudat.models.data.user.composite.DatasetProfile datasetprofile = userManager.generateDatasetProfileModel(datasetEntity.getProfile());
        datasetprofile.setStatus(dataset.getStatus());
        if (datasetEntity.getProperties() != null) {
            JSONObject jObject = new JSONObject(datasetEntity.getProperties());
            Map<String, Object> properties = jObject.toMap();
            datasetprofile.fromJsonObject(properties);
        }
        PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile();
        pagedDatasetProfile.buildPagedDatasetProfile(datasetprofile);
        return pagedDatasetProfile;
    }

    private XWPFDocument getWordDocument(ConfigLoader configLoader, eu.eudat.data.entities.Dataset datasetEntity, VisibilityRuleService visibilityRuleService, Principal principal) throws IOException {
        WordBuilder wordBuilder = new WordBuilder(this.environment);
        DatasetWizardModel dataset = new DatasetWizardModel();
        XWPFDocument document = configLoader.getDatasetDocument();

        eu.eudat.data.entities.DMP dmpEntity = datasetEntity.getDmp();

        if (!dmpEntity.isPublic() && dmpEntity.getUsers().stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()).collect(Collectors.toList()).size() == 0)
            throw new UnauthorisedException();

        wordBuilder.fillFirstPage(dmpEntity, datasetEntity, document);
        wordBuilder.fillFooter(dmpEntity, datasetEntity, document, true);

        int powered_pos = wordBuilder.findPosOfPoweredBy(document);
        XWPFParagraph powered_par = null;
        XWPFParagraph argos_img_par = null;
        if(powered_pos != -1) {
            powered_par = document.getParagraphArray(powered_pos);
            argos_img_par = document.getParagraphArray(powered_pos + 1);
        }

//        wordBuilder.addParagraphContent(datasetEntity.getLabel(), document, ParagraphStyle.HEADER1, BigInteger.ZERO);

        // Space below Dataset title.
//        XWPFParagraph parBreakDataset = document.createParagraph();
//
//        XWPFParagraph datasetTemplateParagraph = document.createParagraph();
//        datasetTemplateParagraph.setStyle("Heading2");
//        XWPFRun runDatasetTemplate1 = datasetTemplateParagraph.createRun();
//        runDatasetTemplate1.setText("Template: ");
//        runDatasetTemplate1.setBold(true);
//        runDatasetTemplate1.setFontSize(12);
//        XWPFRun runDatasetTemplate = datasetTemplateParagraph.createRun();
//        runDatasetTemplate.setText(datasetEntity.getProfile().getLabel());
//        runDatasetTemplate.setColor("2E75B6");
//        runDatasetTemplate.setBold(true);
//        runDatasetTemplate.setFontSize(12);
//
//        wordBuilder.addParagraphContent(datasetEntity.getDescription(), document, ParagraphStyle.HTML, BigInteger.ZERO);

        /*XWPFParagraph externalReferencesParagraph = document.createParagraph();
        externalReferencesParagraph.setStyle("Heading2");
        XWPFRun externalReferencesRun = externalReferencesParagraph.createRun();
        externalReferencesRun.setText("External References");
        externalReferencesRun.setColor("2E75B6");
        externalReferencesRun.setBold(true);
        externalReferencesRun.setFontSize(12);

        wordBuilder.addParagraphContent("Data Repositories", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.getDatasetDataRepositories().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getDatasetDataRepositories().stream().map(DatasetDataRepository::getDataRepository).map(DataRepository::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.TEXT, BigInteger.ZERO);
        }
        wordBuilder.addParagraphContent("External Datasets", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.getDatasetExternalDatasets().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getDatasetExternalDatasets().stream().map(DatasetExternalDataset::getExternalDataset).map(ExternalDataset::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.TEXT, BigInteger.ZERO);
        }
        wordBuilder.addParagraphContent("Registries", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.getRegistries().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getRegistries().stream().map(Registry::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.TEXT, BigInteger.ZERO);
        }
        wordBuilder.addParagraphContent("Services", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.getServices().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getServices().stream().map(DatasetService::getService).map(Service::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.TEXT, BigInteger.ZERO);
        }*/
        /*wordBuilder.addParagraphContent("Tags", document, ParagraphStyle.HEADER3, BigInteger.ZERO);
        if (datasetEntity.().size() > 0) {
            wordBuilder.addParagraphContent(datasetEntity.getServices().stream().map(DatasetService::getService).map(Service::getLabel).collect(Collectors.joining(", "))
                    , document, ParagraphStyle.HEADER4, BigInteger.ZERO);
        }*/

        Map<String, Object> properties = new HashMap<>();
        if (datasetEntity.getProperties() != null) {
            JSONObject jObject = new JSONObject(datasetEntity.getProperties());
            properties = jObject.toMap();
        }

//        wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO);
        PagedDatasetProfile pagedDatasetProfile = getPagedProfile(dataset, datasetEntity);
        visibilityRuleService.setProperties(properties);
        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
        wordBuilder.build(document, pagedDatasetProfile, visibilityRuleService);
        String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", "");
//        File exportFile = new File(label + ".docx");

        // Removes the top empty headings.
//        for (int i = 0; i < 6; i++) {
//            document.removeBodyElement(0);
//        }

        if(powered_pos != -1) {
            document.getLastParagraph().setPageBreak(false);
            document.createParagraph();
            document.setParagraph(powered_par, document.getParagraphs().size() - 1);

            document.createParagraph();
            document.setParagraph(argos_img_par, document.getParagraphs().size() - 1);

            document.removeBodyElement(powered_pos + 1);
            document.removeBodyElement(powered_pos + 1);
        }

        return document;
        //FileOutputStream out = new FileOutputStream(exportFile);
       // document.write(out);
       // out.close();
      //  return exportFile;
    }

    private XWPFDocument getLightWordDocument(ConfigLoader configLoader, DatasetWizardModel dataset, VisibilityRuleService visibilityRuleService) throws IOException {
        WordBuilder wordBuilder = new WordBuilder(this.environment);
        XWPFDocument document = configLoader.getDocument();

        // Space below Dataset title.
        XWPFParagraph parBreakDataset = document.createParagraph();

        Map<String, Object> properties = new HashMap<>();
        if (dataset.getDatasetProfileDefinition() != null) {
            JSONObject jObject = new JSONObject(propertiesModelToString(dataset.getDatasetProfileDefinition()));
            properties = jObject.toMap();
        }

        wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO, 0);
        visibilityRuleService.setProperties(properties);
        visibilityRuleService.buildVisibilityContext(dataset.getDatasetProfileDefinition().getRules());
        wordBuilder.build(document, dataset.getDatasetProfileDefinition(), visibilityRuleService);

        // Removes the top empty headings.
        for (int i = 0; i < 6; i++) {
            document.removeBodyElement(0);
        }

        return document;
        //FileOutputStream out = new FileOutputStream(exportFile);
        // document.write(out);
        // out.close();
        //  return exportFile;
    }

    public FileEnvelope getWordDocumentFile(ConfigLoader configLoader, String id, VisibilityRuleService visibilityRuleService, Principal principal) throws IOException {
        eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
        if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers()
                                                               .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId())
                                                               .collect(Collectors.toList()).size() == 0)
            throw new UnauthorisedException();
        String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", "");
        FileEnvelope exportEnvelope = new FileEnvelope();
        exportEnvelope.setFilename(label + ".docx");
        String uuid = UUID.randomUUID().toString();
        File exportFile = new File(this.environment.getProperty("temp.temp") + uuid + ".docx");
        XWPFDocument document = getWordDocument(configLoader, datasetEntity, visibilityRuleService, principal);
        FileOutputStream out = new FileOutputStream(exportFile);
         document.write(out);
         out.close();
         exportEnvelope.setFile(exportFile);
          return exportEnvelope;
    }

    public String getWordDocumentText (Dataset datasetEntity) throws Exception {
        DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(datasetEntity);
        datasetWizardModel.setDatasetProfileDefinition(this.getPagedProfile(datasetWizardModel, datasetEntity));
        XWPFDocument document = getLightWordDocument(this.configLoader, datasetWizardModel, new VisibilityRuleServiceImpl());
        XWPFWordExtractor extractor = new XWPFWordExtractor(document);
        return extractor.getText();/*.replaceAll("\n\\s*", " ");*/
    }

    public FileEnvelope getXmlDocument(String id, VisibilityRuleService visibilityRuleService, Principal principal) throws InstantiationException, IllegalAccessException, IOException {
        ExportXmlBuilder xmlBuilder = new ExportXmlBuilder();
        DatasetWizardModel dataset = new DatasetWizardModel();
        eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
        if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers()
                         .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId())
                         .collect(Collectors.toList()).size() == 0)
            throw new UnauthorisedException();
        Map<String, Object> properties = new HashMap<>();
        if (datasetEntity.getProperties() != null) {
            JSONObject jobject = new JSONObject(datasetEntity.getProperties());
            properties = jobject.toMap();
        }
        PagedDatasetProfile pagedDatasetProfile = getPagedProfile(dataset, datasetEntity);
        visibilityRuleService.setProperties(properties);
        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());
        File file = xmlBuilder.build(pagedDatasetProfile, datasetEntity.getProfile().getId(), visibilityRuleService, environment);
        FileEnvelope fileEnvelope = new FileEnvelope();
        fileEnvelope.setFile(file);
        String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", "");
        fileEnvelope.setFilename(label);
        return fileEnvelope;
    }

    public eu.eudat.data.entities.Dataset createOrUpdate(DatasetWizardModel datasetWizardModel, Principal principal) throws Exception {
        Boolean sendNotification = false;
        Dataset tempDataset = null;
        DMP dmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(datasetWizardModel.getDmp().getId());
        if (datasetWizardModel.getId() != null) {
           tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(datasetWizardModel.getId());
            if (tempDataset != null) {
            	
            	 Instant dbTime = Instant.ofEpochMilli(tempDataset.getModified().getTime()).truncatedTo(ChronoUnit.SECONDS);
                 Instant modelTime = Instant.ofEpochMilli(datasetWizardModel.getModified().getTime()).truncatedTo(ChronoUnit.SECONDS);
                if (modelTime.toEpochMilli() != dbTime.toEpochMilli()) {
                    throw new Exception("Dataset has been modified already by another user.");
                }
                sendNotification = true;
            }
        } else {
            metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
        }
        if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null)
            throw new Exception("DMP is finalized, therefore Dataset cannot be edited.");
        eu.eudat.data.entities.Dataset dataset = datasetWizardModel.toDataModel();
        dataset.setDmp(dmp);
        dataset.setProperties(propertiesModelToString(datasetWizardModel.getDatasetProfileDefinition()));
        if (this.apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().getClient() != null) {
            this.getTagsFromProfile(datasetWizardModel, dataset);
        }
        if (datasetWizardModel.getStatus() == (int) Dataset.Status.FINALISED.getValue()) {
            String failedField = checkDatasetValidation(dataset);
            if (failedField != null) {
                throw new Exception("Field value of " + failedField + " must be filled.");
            }
        }
        UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
        dataset.setCreator(userInfo);

        createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), dataset);
        createExternalDatasetsIfTheyDontExist(dataset);
        createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), dataset);
        createServicesIfTheyDontExist(dataset);
        Dataset dataset1 = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(dataset);
        datasetWizardModel.setId(dataset1.getId());
        if (datasetWizardModel.getDmp().getGrant() == null) {
            datasetWizardModel.setDmp(new DataManagementPlan().fromDataModelNoDatasets(dataset1.getDmp()));
        }
        dataset1.setProfile(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(datasetWizardModel.getProfile().getId()));
//        datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset1));
        UUID dmpId = dataset1.getDmp().getId();
        dataset1.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
        updateTags(dataset1, datasetWizardModel.getTags());
        if (sendNotification) {
            if (dataset1.getStatus() != Dataset.Status.FINALISED.getValue()) {
                this.sendNotification(dataset1, dataset1.getDmp(), userInfo, NotificationType.DATASET_MODIFIED);
            } else {
                this.sendNotification(dataset1, dataset1.getDmp(), userInfo, NotificationType.DATASET_MODIFIED_FINALISED);
            }
        }

        this.deleteOldFilesAndAddNew(datasetWizardModel, userInfo);


        return dataset1;
    }

    private void deleteOldFilesAndAddNew(DatasetWizardModel datasetWizardModel, UserInfo userInfo) throws JsonProcessingException {
        // Files in DB for this entityId which are NOT DELETED
        List<FileUpload> fileUploads = fileManager.getCurrentFileUploadsForEntityId(datasetWizardModel.getId());
        List<String> fileUploadIds = fileUploads.stream().map(fileUpload -> fileUpload.getId().toString()).collect(Collectors.toList());

        ObjectMapper mapper = new ObjectMapper();
        mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
        String json = mapper.writeValueAsString(datasetWizardModel.getDatasetProfileDefinition());;
        JsonNode propertiesJson = mapper.readTree(json);

        Set<JsonNode> uploadNodes = new HashSet<>();
        uploadNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "upload", true));

        uploadNodes.forEach(node -> {
            JsonNode value = node.get("value");
            if (value != null && !value.toString().equals("\"\"") && !value.toString().equals("null")) {
                String stringValue = value.toString().replaceAll("=", ":");
                JSONObject values = new JSONObject(stringValue);
                Map<String, Object> data = ((JSONObject) values).toMap();

                int index = fileUploadIds.indexOf(data.get("id").toString());
                if(index != -1) {
                    // file in DB is the same as file in the Dataset
                    fileUploadIds.remove(index);
                    fileUploads.remove(index);
                } else {
                    // new file
                    this.fileManager.createFile(data.get("id").toString(), data.get("name").toString(), data.get("type").toString(), datasetWizardModel.getId().toString(), FileUpload.EntityType.DATASET, userInfo);
                }
            }
        });

        // old files in DB that are not contained anymore in the Dataset -> mark them as Deleted
        fileUploads.forEach(fileUpload -> {
            fileManager.markOldFileAsDeleted(fileUpload);
        });
    }

    private void sendNotification(Dataset dataset, DMP dmp, UserInfo user, NotificationType notificationType) {
        List<UserDMP> userDMPS = databaseRepository.getUserDmpDao().asQueryable().where(((builder, root) -> builder.equal(root.get("dmp").get("id"), dmp.getId()))).toList();
        for (UserDMP userDMP : userDMPS) {
            if (!userDMP.getUser().getId().equals(user.getId())) {
                Notification notification = new Notification();
                notification.setUserId(user);
                notification.setType(notificationType);
                notification.setNotifyState(NotifyState.PENDING);
                notification.setIsActive(ActiveStatus.ACTIVE);
                notification.setData("{" +
                        "\"userId\": \"" + userDMP.getUser().getId() + "\"" +
                        ", \"id\": \"" + dataset.getId() + "\"" +
                        ", \"name\": \"" + dataset.getLabel() + "\"" +
                        ", \"path\": \"" + notificationPaths.get(notificationType) + "\"" +
                        "}");
                notification.setCreatedAt(new Date());
                notification.setUpdatedAt(notification.getCreatedAt());
                notification.setContactTypeHint(ContactType.EMAIL);
                notification.setContactHint(userDMP.getUser().getEmail());
                databaseRepository.getNotificationDao().createOrUpdate(notification);
            }
        }

    }

    public String checkDatasetValidation(Dataset dataset) throws Exception {
        List<String> datasetProfileValidators = new LinkedList<>();
        DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(dataset.getProfile().getId());
        DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance();
        DocumentBuilder builder = builderFactory.newDocumentBuilder();
        Document xmlDocument = builder.parse(new ByteArrayInputStream(profile.getDefinition().getBytes()));

        XPath xPath = XPathFactory.newInstance().newXPath();
        String expression = "//validation/@type[.=1]/ancestor::field/@id";
        NodeList nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);

        for (int i = 0; i < nodeList.getLength(); i++) {
            Node node = nodeList.item(i);
            datasetProfileValidators.add(node.getNodeValue());
        }

        expression = "//validation/@type[.=1]/ancestor::fieldSet";
        nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET);


        JSONObject obj = new JSONObject(dataset.getProperties());
        VisibilityRuleService visibilityRuleService = new VisibilityRuleServiceImpl();
        visibilityRuleService.setProperties(obj.toMap());

        dataset.setProfile(profile);
        PagedDatasetProfile pagedDatasetProfile = this.getPagedProfile(new DatasetWizardModel(), dataset);
        visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules());


        String failedField = null;

        for (String validator : datasetProfileValidators) {
            if (obj.has(validator) && isNullOrEmpty(obj.getString(validator)) && isElementVisible(nodeList, validator, visibilityRuleService)) {
                //throw new Exception("Field value of " + validator + " must be filled.");
                failedField = validator;
                break;
            }
        }

        return failedField;
    }

    private boolean isNullOrEmpty(String value) {
        return value == null || value.trim().isEmpty();
    }

    private boolean isElementVisible(NodeList nodeList, String id, VisibilityRuleService visibilityRuleService) {
        Element fieldSet = null;
        for (int i = 0; i < nodeList.getLength(); i++) {
            Node node = nodeList.item(i);
            for (int j = 0; j < node.getChildNodes().getLength(); j++) {
                Node fcnode = node.getChildNodes().item(j);
                if (fcnode.getNodeName().equals("fields")) {
                    for(int k = 0; k < fcnode.getChildNodes().getLength(); k++) {
                        Node scnode = fcnode.getChildNodes().item(k);
                        if (scnode.getNodeName().equals("field") && scnode.getAttributes().getNamedItem("id").getNodeValue().equals(id)) {
                            fieldSet = (Element) node;
                        }
                    }
                }
            }
        }
        if (fieldSet != null) {
            return visibilityRuleService.isElementVisible(id) && visibilityRuleService.isElementVisible(fieldSet.getAttribute("id"));
        } else {
            return visibilityRuleService.isElementVisible(id);
        }
    }

    private String propertiesModelToString(PagedDatasetProfile pagedDatasetProfile) {
        Map<String, Object> values = new LinkedHashMap<>();
        pagedDatasetProfile.toMap(values);
        JSONObject jobject = new JSONObject(values);
        return jobject.toString();
    }

    public void updateTags(Dataset datasetEntity, List<Tag> tags) throws Exception {
       // if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) {
        /*eu.eudat.elastic.entities.Dataset dataset = new eu.eudat.elastic.entities.Dataset();
        dataset.setId(datasetWizardModel.getId().toString());
        if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) {
            DatasetCriteria criteria = new DatasetCriteria();
            criteria.setTags(datasetWizardModel.getTags());
            List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
                                       .filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> datasetWizardModel.getTags().stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
            if (tags.isEmpty()) {
                datasetWizardModel.getTags().forEach(tag -> tag.setId(UUID.randomUUID().toString()));
                dataset.setTags(datasetWizardModel.getTags());
            } else {
                dataset.setTags(tags);
            }
        }
        dataset.setLabel(datasetWizardModel.getLabel());
        dataset.setDescription(datasetWizardModel.getDescription());
        dataset.setTemplate(datasetWizardModel.getProfile());
        dataset.setStatus(datasetWizardModel.getStatus());
        dataset.setDmp(datasetWizardModel.getDmp().getId());
        dataset.setGroup(datasetWizardModel.getDmp().getGroupId());
        dataset.setGrant(datasetWizardModel.getDmp().getGrant().getId());
        if (datasetWizardModel.getDmp().getUsers() != null) {
            dataset.setCollaborators(datasetWizardModel.getDmp().getUsers().stream().map(user -> {
                Collaborator collaborator = new Collaborator();
                collaborator.setId(user.getId().toString());
                collaborator.setName(user.getName());
                return collaborator;
            }).collect(Collectors.toList()));
        }
        DataManagementPlanCriteria dmpCriteria = new DataManagementPlanCriteria();
        dmpCriteria.setAllVersions(true);
        dmpCriteria.setGroupIds(Collections.singletonList(datasetWizardModel.getDmp().getGroupId()));
        apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream()
                  .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> dataset.setLastVersion(dmp.getId().equals(datasetWizardModel.getDmp().getId())));
        apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream().filter(DMP::isPublic)
                  .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> dataset.setLastPublicVersion(dmp.getId().equals(datasetWizardModel.getDmp().getId())));
        if (dataset.getLastVersion() == null) {
            dataset.setLastVersion(true);
        }
        if (dataset.getLastPublicVersion() == null) {
            dataset.setLastPublicVersion(false);
        }
        if (datasetWizardModel.getDmp().getOrganisations() != null) {
            dataset.setOrganizations(datasetWizardModel.getDmp().getOrganisations().stream().map(org -> {
                Organization organization = new Organization();
                organization.setId(org.getId());
                organization.setName(org.getName());
                return organization;
            }).collect(Collectors.toList()));
        }
        dataset.setPublic(datasetWizardModel.getDmp().getPublic());
        dataset.setGrantStatus(datasetWizardModel.getDmp().getGrant().getStatus());
        dataset.setFormData(this.getWordDocumentText(datasetWizardModel));*/
        DatasetMapper mapper = new DatasetMapper(apiContext, this);
        eu.eudat.elastic.entities.Dataset dataset = mapper.toElastic(datasetEntity, tags);
        apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(dataset);
       // }
    }



    private void createRegistriesIfTheyDontExist(RegistryDao registryDao, eu.eudat.data.entities.Dataset dataset) {
        if (dataset.getRegistries() != null && !dataset.getRegistries().isEmpty()) {
            for (eu.eudat.data.entities.Registry registry : dataset.getRegistries()) {
                RegistryCriteria criteria = new RegistryCriteria();
                criteria.setLike(registry.getReference());
                List<eu.eudat.data.entities.Registry> entries = registryDao.getWithCriteria(criteria).toList();
                if (entries != null && !entries.isEmpty()) registry.setId(entries.get(0).getId());
                else {
                    registry.setCreated(new Date());
                    registryDao.createOrUpdate(registry);
                }
            }
        }
    }

    private void createDataRepositoriesIfTheyDontExist(DataRepositoryDao dataRepositoryDao, eu.eudat.data.entities.Dataset dataset) {
        if (dataset.getDatasetDataRepositories() != null && !dataset.getDatasetDataRepositories().isEmpty()) {
            for (eu.eudat.data.entities.DatasetDataRepository datasetDataRepository : dataset.getDatasetDataRepositories()) {
                DataRepositoryCriteria criteria = new DataRepositoryCriteria();
                criteria.setLike(datasetDataRepository.getDataRepository().getReference());
                List<eu.eudat.data.entities.DataRepository> entries = dataRepositoryDao.getWithCriteria(criteria).toList();
                if (entries != null && !entries.isEmpty()) {
                    datasetDataRepository.getDataRepository().setId(entries.get(0).getId());
                    datasetDataRepository.setDataset(dataset);
                    dataset.getDatasetDataRepositories().add(datasetDataRepository);
                } else {
                    datasetDataRepository.getDataRepository().setId(UUID.randomUUID());
                    DataRepository dataRepository = dataRepositoryDao.createOrUpdate(datasetDataRepository.getDataRepository());
                    datasetDataRepository.setDataset(dataset);
                    datasetDataRepository.setDataRepository(dataRepository);
                    dataset.getDatasetDataRepositories().add(datasetDataRepository);
                }
            }
        }
    }

    private void createServicesIfTheyDontExist(eu.eudat.data.entities.Dataset dataset) {
        if (dataset.getServices() != null && !dataset.getServices().isEmpty()) {
            for (DatasetService service : dataset.getServices()) {
                ServiceCriteria criteria = new ServiceCriteria();
                criteria.setLike(service.getService().getReference());
                List<eu.eudat.data.entities.Service> entries = databaseRepository.getServiceDao().getWithCriteria(criteria).toList();
                if (entries != null && !entries.isEmpty()) {
                    service.setDataset(dataset);
                    service.getService().setCreated(new Date());
                    service.setService(service.getService());
                    this.databaseRepository.getServiceDao().createOrUpdate(service.getService());
                    dataset.getServices().add(service);
                }
            }
        }
    }

    private void createExternalDatasetsIfTheyDontExist(eu.eudat.data.entities.Dataset dataset) {
        if (dataset.getDatasetExternalDatasets() != null && !dataset.getDatasetExternalDatasets().isEmpty()) {
            for (eu.eudat.data.entities.DatasetExternalDataset datasetExternalDataset : dataset.getDatasetExternalDatasets()) {
                ExternalDatasetCriteria criteria = new ExternalDatasetCriteria();
                criteria.setLike(datasetExternalDataset.getExternalDataset().getReference());
                List<eu.eudat.data.entities.ExternalDataset> entries = databaseRepository.getExternalDatasetDao().getWithCriteria(criteria).toList();
                if (entries != null && !entries.isEmpty()) {
                    datasetExternalDataset.getExternalDataset().setId(entries.get(0).getId());
                    datasetExternalDataset.setDataset(dataset);
                    dataset.getDatasetExternalDatasets().add(datasetExternalDataset);
                } else {
                    datasetExternalDataset.getExternalDataset().setId(UUID.randomUUID());
                    datasetExternalDataset.setDataset(dataset);
                    ExternalDataset externalDataset = databaseRepository.getExternalDatasetDao().createOrUpdate(datasetExternalDataset.getExternalDataset());
                    datasetExternalDataset.setExternalDataset(externalDataset);
                    dataset.getDatasetExternalDatasets().add(datasetExternalDataset);
                }
            }
        }
    }

    public void makePublic(DatasetDao datasetDao, UUID id) throws Exception {
        eu.eudat.data.entities.Dataset dataset = datasetDao.find(id);
        if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue())
            throw new Exception("You cannot make public a Dataset That Has not Been Finalised");
        datasetDao.createOrUpdate(dataset);
        metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
    }

    public ResponseEntity<byte[]> getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException {
        FileEnvelope envelope = getXmlDocument(id, visibilityRuleService, principal);
        InputStream resource = new FileInputStream(envelope.getFile());
        logger.info("Mime Type of " + envelope.getFilename() + " is " +
                new MimetypesFileTypeMap().getContentType(envelope.getFile()));
        HttpHeaders responseHeaders = new HttpHeaders();
        responseHeaders.setContentLength(envelope.getFile().length());
        responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM);
        String fileName = envelope.getFilename().replace(" ", "_").replace(",", "_");
        responseHeaders.set("Content-Disposition", "attachment;filename=" + fileName + ".xml");
        responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition");
        responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type");

        byte[] content = org.apache.poi.util.IOUtils.toByteArray(resource);
        resource.close();
        Files.deleteIfExists(envelope.getFile().toPath());

        return new ResponseEntity<>(content,
                responseHeaders,
                HttpStatus.OK);
    }

    public eu.eudat.data.entities.Dataset createDatasetFromXml(MultipartFile importFile, String dmpId, String datasetProfileId, Principal principal) throws JAXBException, IOException {
        DatasetImportPagedDatasetProfile importModel = new DatasetImportPagedDatasetProfile();
        JAXBContext jaxbContext;

        // Parses XML into DatasetImport Model.
        try {
            InputStream in = importFile.getInputStream();
            jaxbContext = JAXBContext.newInstance(DatasetImportPagedDatasetProfile.class);
            Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller();
            DatasetImportPagedDatasetProfile datasetImport = (DatasetImportPagedDatasetProfile) jaxbUnmarshaller.unmarshal(in);
            importModel = datasetImport;
        } catch (IOException e) {
            logger.error(e.getMessage(), e);
        }

        // Checks if XML datasetProfileId GroupId matches the one selected.
        try {
            eu.eudat.data.entities.DatasetProfile importDatasetProfile = databaseRepository.getDatasetProfileDao().find(UUID.fromString(importModel.getDatasetProfileId()));
            eu.eudat.data.entities.DatasetProfile latestVersionDatasetProfile = databaseRepository.getDatasetProfileDao().find(UUID.fromString(datasetProfileId));
            if (latestVersionDatasetProfile.getGroupId() != importDatasetProfile.getGroupId()) {
                throw new Exception();
            }
        } catch (Exception e) {
            logger.error(e.getMessage(), e);
            return null;
        }

        // Creates the Hash Map to place the values of the data set.
        Map<String, String> importMap = importModel.getPages().stream()
                .flatMap(s -> s.getSections().getSection().stream()
                        .flatMap(cFields -> cFields.getCompositeFields().stream()
                                .flatMap(cField -> cField.getCompositeField().stream()
                                        .filter(Objects::nonNull)
                                        .flatMap(fields -> fields.getFields().stream()
                                                .flatMap(field -> field.getField().stream()
                                                        .filter(f -> f.getValue() != null)
                                                )))))
                .collect(Collectors.toMap(DatasetImportField::getId, DatasetImportField::getValue));

        // Transforms map into json file.
        JSONObject jsonDatasetProperties = new JSONObject(importMap);

        // Creates the entity data set to save.
        eu.eudat.data.entities.Dataset entity = new Dataset();
        entity.setProperties(jsonDatasetProperties.toString());
        entity.setLabel(importFile.getOriginalFilename());
        DMP dmp = new DMP();
        dmp.setId(UUID.fromString(dmpId));
        entity.setDmp(dmp);
        entity.setStatus((short) 0);
        entity.setCreated(new Date());
        entity.setModified(new Date());
        DatasetProfile profile = new DatasetProfile();
        profile.setId(UUID.fromString(datasetProfileId));
        entity.setProfile(profile);

        UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
        entity.setCreator(userInfo);

        updateTagsXmlImportDataset(apiContext.getOperationsContext().getElasticRepository().getDatasetRepository(), entity);
        createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), entity);
        createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), entity);
        createServicesIfTheyDontExist(entity);
        createExternalDatasetsIfTheyDontExist(entity);

        metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
        return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity);
    }

    public void updateTagsXmlImportDataset(DatasetRepository datasetRepository, Dataset dataset) throws IOException {
        // TODO: When tags functionality return.
    }

    public DatasetWizardModel datasetUpdateProfile(String id) {
        DatasetWizardModel dataset = new DatasetWizardModel();
        eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
        eu.eudat.elastic.entities.Dataset datasetElastic;
        try {
            datasetElastic = datasetRepository.exists() ?
                    datasetRepository.findDocument(id) : new eu.eudat.elastic.entities.Dataset();
        } catch (Exception ex) {
            logger.warn(ex.getMessage());
            datasetElastic = null;
        }
        dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity));
        dataset.fromDataModel(datasetEntity);

        // Creates the Criteria to get all version of DatasetProfile in question.
        DatasetProfileCriteria profileCriteria = new DatasetProfileCriteria();
        UUID profileId = datasetEntity.getProfile().getGroupId();
        List<UUID> uuidList = new LinkedList<>();
        uuidList.add(profileId);
        profileCriteria.setGroupIds(uuidList);

        // Gets the latest version of the datasetProfile.
        eu.eudat.data.entities.DatasetProfile item = databaseRepository.getDatasetProfileDao().getWithCriteria(profileCriteria).getSingle();

        // Sets the latest version of dataet Profile to the Dataset in question.
        dataset.setDatasetProfileDefinition(getLatestDatasetProfile(datasetEntity, item));
        dataset.setProfile(new DatasetProfileOverviewModel().fromDataModel(item));

        // Now at latest version.
        dataset.setIsProfileLatestVersion(true);

        if (datasetElastic != null && datasetElastic.getTags() != null && !datasetElastic.getTags().isEmpty()) {
            dataset.setTags(datasetElastic.getTags());
        }
        /*if (datasetElastic != null && datasetElastic.getLabel() != null && !datasetElastic.getLabel().isEmpty()) {
            dataset.setLabel(datasetElastic.getLabel());
        }*/
        return dataset;
    }

    public PagedDatasetProfile getLatestDatasetProfile(Dataset datasetEntity, DatasetProfile profile) {
        eu.eudat.models.data.user.composite.DatasetProfile datasetprofile = userManager.generateDatasetProfileModel(profile);
        datasetprofile.setStatus(datasetEntity.getStatus());
        if (datasetEntity.getProperties() != null) {
            JSONObject jobject = new JSONObject(datasetEntity.getProperties());
            Map<String, Object> properties = jobject.toMap();
            datasetprofile.fromJsonObject(properties);
        }
        PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile();
        pagedDatasetProfile.buildPagedDatasetProfile(datasetprofile);
        return pagedDatasetProfile;
    }

    public DataTableData<DatasetProfileListingModel> getDatasetProfilesUsedByDatasets(DatasetProfileTableRequestItem datasetProfileTableRequestItem, Principal principal) {
        datasetProfileTableRequestItem.getCriteria().setFilter(DatasetProfileCriteria.DatasetProfileFilter.Datasets.getValue());
        datasetProfileTableRequestItem.getCriteria().setUserId(principal.getId());

        QueryableList<DatasetProfile> items = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(datasetProfileTableRequestItem.getCriteria());
        List<DatasetProfileListingModel> listingModels = items.select(item -> new DatasetProfileListingModel().fromDataModel(item));

        DataTableData<DatasetProfileListingModel> data = new DataTableData<>();
        data.setData(listingModels);
        data.setTotalCount((long) listingModels.size());

        return data;
    }

    public void generateIndex(Principal principal) {
        if (principal.getAuthorities().contains(Authorities.ADMIN.getValue())) {
            this.apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList();
            List<Dataset> datasetEntities = new ArrayList<>(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().asQueryable().toList());
            datasetEntities.forEach(datasetEntity -> {
                try {
                    eu.eudat.elastic.entities.Dataset dataset = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(datasetEntity.getId().toString());
                    UUID dmpId = datasetEntity.getDmp().getId();
                    datasetEntity.getDmp().setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()));
                    updateTags(datasetEntity, dataset != null ? dataset.getTags() : null);
                } catch (Exception e) {
                    logger.error(e.getMessage(), e);
                }
            });
        }
    }

    public void clearIndex(Principal principal) {
        if (principal.getAuthorities().contains(Authorities.ADMIN.getValue())) {
            try {
                this.apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().clear();
            } catch (IOException e) {
                logger.error(e.getMessage(), e);
            }
        }
    }

    public void getTagsFromProfile(DatasetWizardModel wizardModel, Dataset dataset) throws IOException {
        dataset.setProfile(apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(dataset.getProfile().getId()));
        wizardModel.setDatasetProfileDefinition(this.getPagedProfile(wizardModel, dataset));
        ObjectMapper mapper = new ObjectMapper();
        String json = mapper.writeValueAsString(wizardModel.getDatasetProfileDefinition());
        JsonNode propertiesJson = mapper.readTree(json);
        DatasetCriteria criteria = new DatasetCriteria();
        criteria.setHasTags(true);
        List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList());
        Set<JsonNode> tagNodes = new HashSet<>();
        tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "tags", true));
        tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "rdaProperty", "dataset.keyword"));
        if(wizardModel.getTags() == null){
            wizardModel.setTags(new ArrayList<>());
        }
        if (!tagNodes.isEmpty()) {
            tagNodes.forEach(node -> {
                JsonNode value = node.get("value");
                if (!value.toString().equals("\"\"") && !value.toString().equals("null") && value.toString().startsWith("[")) {
                    String stringValue = value.toString().replaceAll("=", ":");
                    JSONArray values = new JSONArray(stringValue);
                    values.iterator().forEachRemaining(element -> {
                        Map<String, Object> data = ((JSONObject) element).toMap();
                        this.addTag(tags, wizardModel.getTags(), data.get("id").toString(), data.get("name").toString());
                    });
                } else {
                    List<String> values = Arrays.asList(value.textValue().split(", "));
                    List<Tag> tagValues = values.stream().map(stringValue -> new Tag(stringValue, stringValue)).collect(Collectors.toList());
                    tagValues.iterator().forEachRemaining(tag -> {
                        this.addTag(tags, wizardModel.getTags(), tag.getId(), tag.getName());
                    });
                }
            });
        }
    }

    private void addTag(List<Tag> srcTags, List<Tag> dstTags, String id, String name) {
        Tag tag = new Tag();
        if(srcTags.stream().anyMatch(intag -> intag.getName().equals(name))) {
            tag = srcTags.stream().filter(intag -> intag.getName().equals(name)).findFirst().get();
        } else {
            tag.setName(name);
            tag.setId(id);
        }
        if (dstTags.stream().noneMatch(intag -> intag.getName().equals(name))) {
            dstTags.add(tag);
        }
    }

    @Transactional
    private DatasetListingModel mapModel(Dataset item) {
        /*if (item.getProfile() == null)
            return null;*/
        DatasetListingModel listingModel = new DatasetListingModel().fromDataModel(item);
        /*DatasetProfileCriteria criteria = new DatasetProfileCriteria();
        criteria.setGroupIds(Collections.singletonList(item.getProfile().getGroupId()));
        List<DatasetProfile> profiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
        boolean islast = false;
        if (!profiles.isEmpty()) {
            profiles = profiles.stream().sorted(Comparator.comparing(DatasetProfile::getVersion)).collect(Collectors.toList());
            islast = profiles.get(0).getId().equals(item.getProfile().getId());
        }
        listingModel.setProfileLatestVersion(islast);*/
        return listingModel;
    }
}
