package eu.eudat.logic.managers; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import eu.eudat.data.dao.criteria.*; import eu.eudat.data.dao.entities.DataRepositoryDao; import eu.eudat.data.dao.entities.DatasetDao; import eu.eudat.data.dao.entities.RegistryDao; import eu.eudat.data.entities.*; import eu.eudat.data.enumeration.notification.ActiveStatus; import eu.eudat.data.enumeration.notification.ContactType; import eu.eudat.data.enumeration.notification.NotificationType; import eu.eudat.data.enumeration.notification.NotifyState; import eu.eudat.data.query.items.table.dataset.DatasetPublicTableRequest; import eu.eudat.data.query.items.table.dataset.DatasetTableRequest; import eu.eudat.data.query.items.table.datasetprofile.DatasetProfileTableRequestItem; import eu.eudat.elastic.criteria.DatasetCriteria; import eu.eudat.elastic.entities.Tag; import eu.eudat.elastic.repository.DatasetRepository; import eu.eudat.exceptions.security.ForbiddenException; import eu.eudat.exceptions.security.UnauthorisedException; import eu.eudat.logic.builders.BuilderFactory; import eu.eudat.logic.builders.entity.UserInfoBuilder; import eu.eudat.logic.mapper.elastic.DatasetMapper; import eu.eudat.logic.proxy.config.configloaders.ConfigLoader; import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.services.forms.VisibilityRuleService; import eu.eudat.logic.services.operations.DatabaseRepository; import eu.eudat.logic.utilities.documents.helpers.FileEnvelope; import eu.eudat.logic.utilities.documents.types.ParagraphStyle; import eu.eudat.logic.utilities.documents.word.WordBuilder; import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder; import eu.eudat.logic.utilities.helpers.StreamDistinctBy; import eu.eudat.logic.utilities.json.JsonSearcher; import eu.eudat.models.HintedModelFactory; import eu.eudat.models.data.dataset.DatasetOverviewModel; import eu.eudat.models.data.datasetImport.DatasetImportField; import eu.eudat.models.data.datasetImport.DatasetImportPagedDatasetProfile; import eu.eudat.models.data.datasetprofile.DatasetProfileListingModel; import eu.eudat.models.data.datasetprofile.DatasetProfileOverviewModel; import eu.eudat.models.data.datasetwizard.DatasetWizardModel; import eu.eudat.models.data.dmp.AssociatedProfile; import eu.eudat.models.data.dmp.DataManagementPlan; import eu.eudat.models.data.helpers.common.DataTableData; import eu.eudat.models.data.listingmodels.DatasetListingModel; import eu.eudat.models.data.security.Principal; import eu.eudat.models.data.user.composite.PagedDatasetProfile; import eu.eudat.queryable.QueryableList; import eu.eudat.types.Authorities; import org.apache.commons.io.IOUtils; import org.apache.poi.xwpf.extractor.XWPFWordExtractor; import org.apache.poi.xwpf.usermodel.XWPFDocument; import org.apache.poi.xwpf.usermodel.XWPFParagraph; import org.apache.poi.xwpf.usermodel.XWPFRun; import org.json.JSONArray; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.core.io.FileSystemResource; import org.springframework.http.*; import org.springframework.stereotype.Component; import org.springframework.util.LinkedMultiValueMap; import org.springframework.web.client.RestTemplate; import org.springframework.web.multipart.MultipartFile; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import javax.activation.MimetypesFileTypeMap; import javax.persistence.criteria.Join; import javax.persistence.criteria.JoinType; import javax.transaction.Transactional; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; import java.io.*; import java.math.BigInteger; import java.nio.file.Files; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @Component public class DatasetManager { private static final Logger logger = LoggerFactory.getLogger(DatasetManager.class); private final Map notificationPaths = Stream.of(new Object[][] { {NotificationType.DATASET_MODIFIED, "/datasets/edit"}, {NotificationType.DATASET_MODIFIED_FINALISED, "/datasets/edit"} }).collect(Collectors.toMap(data -> (NotificationType) data[0], data -> (String) data[1])); private ApiContext apiContext; private DatabaseRepository databaseRepository; private DatasetRepository datasetRepository; private BuilderFactory builderFactory; private UserManager userManager; private ConfigLoader configLoader; private Environment environment; @Autowired public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository(); this.builderFactory = apiContext.getOperationsContext().getBuilderFactory(); this.userManager = userManager; this.configLoader = configLoader; this.environment = environment; } public DataTableData getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception { DatasetCriteria datasetCriteria = new DatasetCriteria(); datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike()); datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetTemplates()); if (datasetTableRequest.getCriteria().getStatus() != null) { datasetCriteria.setStatus(datasetTableRequest.getCriteria().getStatus().shortValue()); } datasetCriteria.setDmps(datasetTableRequest.getCriteria().getDmpIds()); datasetCriteria.setGroupIds(datasetTableRequest.getCriteria().getGroupIds()); datasetCriteria.setGrants(datasetTableRequest.getCriteria().getGrants()); datasetCriteria.setCollaborators(datasetTableRequest.getCriteria().getCollaborators()); datasetCriteria.setAllowAllVersions(datasetTableRequest.getCriteria().getAllVersions()); datasetCriteria.setOrganiztions(datasetTableRequest.getCriteria().getOrganisations()); datasetCriteria.setTags(datasetTableRequest.getCriteria().getTags()); if (datasetTableRequest.getCriteria().getIsPublic() != null) { datasetCriteria.setPublic(datasetTableRequest.getCriteria().getIsPublic()); } if (datasetTableRequest.getCriteria().getGrantStatus() != null) { datasetCriteria.setGrantStatus(datasetTableRequest.getCriteria().getGrantStatus()); } List datasets; try { datasets = datasetRepository.exists() ? datasetRepository.query(datasetCriteria) : null; } catch (Exception ex) { logger.warn(ex.getMessage(), ex); datasets = null; } UserInfo userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); // QueryableList items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); QueryableList items; if (datasets != null) { if (!datasets.isEmpty()) { items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class)); List finalDatasets = datasets; items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))); } else { items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); //items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()})); } } else { items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); } List roles = new LinkedList<>(); QueryableList pagedItems; QueryableList authItems; if (!datasetTableRequest.getCriteria().getIsPublic()) { if (principal.getId() == null) { throw new UnauthorisedException("You are not allowed to access those datasets"); } if (datasetTableRequest.getCriteria().getRole() != null) roles.add(datasetTableRequest.getCriteria().getRole()); authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles); pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest); } else { if (principal.getId() != null && datasetTableRequest.getCriteria().getRole() != null) { items.where((builder, root) -> { Join userJoin = root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT); return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole())); }); } String[] strings = new String[1]; //strings[0] = "-dmp:publishedAt|join|"; //datasetTableRequest.getOrderings().setFields(strings); authItems = items; pagedItems = PaginationManager.applyPaging(items, datasetTableRequest); } DataTableData dataTable = new DataTableData<>(); dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList())); dataTable.setTotalCount(authItems.count()); //CompletableFuture.allOf(itemsFuture, countFuture).join(); return dataTable; } public DataTableData getPaged(DatasetPublicTableRequest datasetTableRequest, Principal principal) throws Exception { DatasetCriteria datasetCriteria = new DatasetCriteria(); datasetCriteria.setLike(datasetTableRequest.getCriteria().getLike()); datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile()); datasetCriteria.setDmps(datasetTableRequest.getCriteria().getDmpIds()); datasetCriteria.setGrants(datasetTableRequest.getCriteria().getGrants()); List datasets; try { datasets = datasetRepository.exists() ? datasetRepository.query(datasetCriteria) : new LinkedList<>(); } catch (Exception ex) { logger.warn(ex.getMessage()); datasets = null; } datasetTableRequest.setQuery(databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class))); QueryableList items = datasetTableRequest.applyCriteria(); /*QueryableList items; if (datasets != null) { if (!datasets.isEmpty()) { items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class)); List finalDatasets = datasets; items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))); } else items = datasetTableRequest.applyCriteria(); items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()})); } else { items = datasetTableRequest.applyCriteria(); }*/ if (principal.getId() != null && datasetTableRequest.getCriteria().getRole() != null) { items.where((builder, root) -> { Join userJoin = root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT); return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole())); }); } String[] strings = new String[1]; strings[0] = "-dmp:publishedAt|join|"; datasetTableRequest.getOrderings().setFields(strings); QueryableList pagedItems = PaginationManager.applyPaging(items, datasetTableRequest); DataTableData dataTable = new DataTableData<>(); CompletableFuture> itemsFuture = pagedItems. selectAsync(this::mapModel).whenComplete((resultList, throwable) -> { dataTable.setData(resultList.stream().filter(Objects::nonNull).collect(Collectors.toList())); }); CompletableFuture countFuture = pagedItems.countAsync().whenComplete((count, throwable) -> { dataTable.setTotalCount(count); }); CompletableFuture.allOf(itemsFuture, countFuture).join(); return dataTable; } public DatasetWizardModel getSingle(String id, Principal principal) { DatasetWizardModel dataset = new DatasetWizardModel(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); if (datasetEntity.getDmp().getUsers() .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()) .collect(Collectors.toList()).size() == 0 && !datasetEntity.getDmp().isPublic()) throw new UnauthorisedException(); eu.eudat.elastic.entities.Dataset datasetElastic; try { datasetElastic = datasetRepository.exists() ? datasetRepository.findDocument(id) : new eu.eudat.elastic.entities.Dataset(); } catch (Exception ex) { logger.warn(ex.getMessage()); datasetElastic = null; } dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity)); dataset.fromDataModel(datasetEntity); // Creates the Criteria to get all version of DatasetProfile in question. DatasetProfileCriteria profileCriteria = new DatasetProfileCriteria(); UUID profileId = datasetEntity.getProfile().getGroupId(); List uuidList = new LinkedList<>(); uuidList.add(profileId); profileCriteria.setGroupIds(uuidList); profileCriteria.setAllVersions(true); List profileVersions = databaseRepository.getDatasetProfileDao().getWithCriteria(profileCriteria) .orderBy(((builder, root) -> builder.desc(root.get("version")))) .toList(); List profileVersionsIncluded = new LinkedList<>(); // Iterate through the versions and remove those that are not included in the DMP of the dataset in question. for (DatasetProfile version : profileVersions) { for (AssociatedProfile p : dataset.getDmp().getProfiles()) { if (version.getId().toString().equals(p.getId().toString())) { profileVersionsIncluded.add(version); } } } // Sort the list with the included Versions. Stream sorted = profileVersionsIncluded.stream().sorted(Comparator.comparing(DatasetProfile::getVersion).reversed()); // Make the Stream into List and get the first item. List profiles = sorted.collect(Collectors.toList()); if (profiles.isEmpty()) throw new NoSuchElementException("No profiles found for the specific Dataset"); DatasetProfile profile = profiles.get(0); // Check if the dataset is on the latest Version. boolean latestVersion = profile.getVersion().toString().equals(datasetEntity.getProfile().getVersion().toString()); dataset.setIsProfileLatestVersion(latestVersion); if (datasetElastic != null && datasetElastic.getTags() != null && !datasetElastic.getTags().isEmpty()) { dataset.setTags(datasetElastic.getTags()); } /*if (datasetElastic != null && datasetElastic.getLabel() != null && !datasetElastic.getLabel().isEmpty()) { dataset.setLabel(datasetElastic.getLabel()); }*/ return dataset; } public DatasetWizardModel getSinglePublic(String id) throws Exception { DatasetWizardModel dataset = new DatasetWizardModel(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().isPublicDataset(UUID.fromString(id)); if (datasetEntity != null && datasetEntity.getStatus() == 1 && datasetEntity.getDmp().getStatus() == 1) { dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity)); dataset.fromDataModel(datasetEntity); return dataset; } else { throw new Exception("Selected dataset is not public"); } } public DatasetOverviewModel getOverviewSingle(String id, Principal principal, boolean isPublic) throws Exception { Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id)); if (datasetEntity.getStatus() == Dataset.Status.DELETED.getValue()) { throw new Exception("Dataset is deleted."); } if (!isPublic && principal == null) { throw new UnauthorisedException(); } else if (!isPublic && datasetEntity.getDmp().getUsers() .stream().noneMatch(userInfo -> userInfo.getUser().getId() == principal.getId())) { throw new UnauthorisedException(); } else if (isPublic && !datasetEntity.getDmp().isPublic()) { throw new ForbiddenException("Selected Dataset is not public"); } DatasetOverviewModel dataset = new DatasetOverviewModel(); dataset.fromDataModel(datasetEntity); return dataset; } public PagedDatasetProfile getPagedProfile(DatasetWizardModel dataset, eu.eudat.data.entities.Dataset datasetEntity) { eu.eudat.models.data.user.composite.DatasetProfile datasetprofile = userManager.generateDatasetProfileModel(datasetEntity.getProfile()); datasetprofile.setStatus(dataset.getStatus()); if (datasetEntity.getProperties() != null) { JSONObject jObject = new JSONObject(datasetEntity.getProperties()); Map properties = jObject.toMap(); datasetprofile.fromJsonObject(properties); } PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile(); pagedDatasetProfile.buildPagedDatasetProfile(datasetprofile); return pagedDatasetProfile; } private XWPFDocument getWordDocument(ConfigLoader configLoader, eu.eudat.data.entities.Dataset datasetEntity, VisibilityRuleService visibilityRuleService) throws IOException { WordBuilder wordBuilder = new WordBuilder(); DatasetWizardModel dataset = new DatasetWizardModel(); XWPFDocument document = configLoader.getDocument(); wordBuilder.addParagraphContent(datasetEntity.getLabel(), document, ParagraphStyle.HEADER1, BigInteger.ZERO); // Space below Dataset title. XWPFParagraph parBreakDataset = document.createParagraph(); XWPFParagraph datasetTemplateParagraph = document.createParagraph(); datasetTemplateParagraph.setStyle("Heading2"); XWPFRun runDatasetTemplate1 = datasetTemplateParagraph.createRun(); runDatasetTemplate1.setText("Template: "); runDatasetTemplate1.setBold(true); runDatasetTemplate1.setFontSize(12); XWPFRun runDatasetTemplate = datasetTemplateParagraph.createRun(); runDatasetTemplate.setText(datasetEntity.getProfile().getLabel()); runDatasetTemplate.setColor("2E75B6"); runDatasetTemplate.setBold(true); runDatasetTemplate.setFontSize(12); /*XWPFParagraph externalReferencesParagraph = document.createParagraph(); externalReferencesParagraph.setStyle("Heading2"); XWPFRun externalReferencesRun = externalReferencesParagraph.createRun(); externalReferencesRun.setText("External References"); externalReferencesRun.setColor("2E75B6"); externalReferencesRun.setBold(true); externalReferencesRun.setFontSize(12); wordBuilder.addParagraphContent("Data Repositories", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.getDatasetDataRepositories().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getDatasetDataRepositories().stream().map(DatasetDataRepository::getDataRepository).map(DataRepository::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.TEXT, BigInteger.ZERO); } wordBuilder.addParagraphContent("External Datasets", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.getDatasetExternalDatasets().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getDatasetExternalDatasets().stream().map(DatasetExternalDataset::getExternalDataset).map(ExternalDataset::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.TEXT, BigInteger.ZERO); } wordBuilder.addParagraphContent("Registries", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.getRegistries().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getRegistries().stream().map(Registry::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.TEXT, BigInteger.ZERO); } wordBuilder.addParagraphContent("Services", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.getServices().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getServices().stream().map(DatasetService::getService).map(Service::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.TEXT, BigInteger.ZERO); }*/ /*wordBuilder.addParagraphContent("Tags", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getServices().stream().map(DatasetService::getService).map(Service::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.HEADER4, BigInteger.ZERO); }*/ Map properties = new HashMap<>(); if (datasetEntity.getProperties() != null) { JSONObject jObject = new JSONObject(datasetEntity.getProperties()); properties = jObject.toMap(); } wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO); PagedDatasetProfile pagedDatasetProfile = getPagedProfile(dataset, datasetEntity); visibilityRuleService.setProperties(properties); visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules()); wordBuilder.build(document, pagedDatasetProfile, visibilityRuleService); String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", ""); // File exportFile = new File(label + ".docx"); // Removes the top empty headings. for (int i = 0; i < 6; i++) { document.removeBodyElement(0); } return document; //FileOutputStream out = new FileOutputStream(exportFile); // document.write(out); // out.close(); // return exportFile; } private XWPFDocument getLightWordDocument(ConfigLoader configLoader, DatasetWizardModel dataset, VisibilityRuleService visibilityRuleService) throws IOException { WordBuilder wordBuilder = new WordBuilder(); XWPFDocument document = configLoader.getDocument(); // Space below Dataset title. XWPFParagraph parBreakDataset = document.createParagraph(); Map properties = new HashMap<>(); if (dataset.getDatasetProfileDefinition() != null) { JSONObject jObject = new JSONObject(propertiesModelToString(dataset.getDatasetProfileDefinition())); properties = jObject.toMap(); } wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO); visibilityRuleService.setProperties(properties); visibilityRuleService.buildVisibilityContext(dataset.getDatasetProfileDefinition().getRules()); wordBuilder.build(document, dataset.getDatasetProfileDefinition(), visibilityRuleService); // Removes the top empty headings. for (int i = 0; i < 6; i++) { document.removeBodyElement(0); } return document; //FileOutputStream out = new FileOutputStream(exportFile); // document.write(out); // out.close(); // return exportFile; } public FileEnvelope getWordDocumentFile(ConfigLoader configLoader, String id, VisibilityRuleService visibilityRuleService, Principal principal) throws IOException { eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers() .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()) .collect(Collectors.toList()).size() == 0) throw new UnauthorisedException(); String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", ""); FileEnvelope exportEnvelope = new FileEnvelope(); exportEnvelope.setFilename(label + ".docx"); String uuid = UUID.randomUUID().toString(); File exportFile = new File(this.environment.getProperty("temp.temp") + uuid + ".docx"); XWPFDocument document = getWordDocument(configLoader, datasetEntity, visibilityRuleService); FileOutputStream out = new FileOutputStream(exportFile); document.write(out); out.close(); exportEnvelope.setFile(exportFile); return exportEnvelope; } public String getWordDocumentText (Dataset datasetEntity) throws Exception { DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(datasetEntity); datasetWizardModel.setDatasetProfileDefinition(this.getPagedProfile(datasetWizardModel, datasetEntity)); XWPFDocument document = getLightWordDocument(this.configLoader, datasetWizardModel, this.apiContext.getUtilitiesService().getVisibilityRuleService()); XWPFWordExtractor extractor = new XWPFWordExtractor(document); return extractor.getText();/*.replaceAll("\n\\s*", " ");*/ } public FileEnvelope getXmlDocument(String id, VisibilityRuleService visibilityRuleService, Principal principal) throws InstantiationException, IllegalAccessException, IOException { ExportXmlBuilder xmlBuilder = new ExportXmlBuilder(); DatasetWizardModel dataset = new DatasetWizardModel(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers() .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()) .collect(Collectors.toList()).size() == 0) throw new UnauthorisedException(); Map properties = new HashMap<>(); if (datasetEntity.getProperties() != null) { JSONObject jobject = new JSONObject(datasetEntity.getProperties()); properties = jobject.toMap(); } PagedDatasetProfile pagedDatasetProfile = getPagedProfile(dataset, datasetEntity); visibilityRuleService.setProperties(properties); visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules()); File file = xmlBuilder.build(pagedDatasetProfile, datasetEntity.getProfile().getId(), visibilityRuleService, environment); FileEnvelope fileEnvelope = new FileEnvelope(); fileEnvelope.setFile(file); String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", ""); fileEnvelope.setFilename(label); return fileEnvelope; } public File convertToPDF(FileEnvelope file, Environment environment) throws IOException, InterruptedException { LinkedMultiValueMap map = new LinkedMultiValueMap<>(); String uuid = UUID.randomUUID().toString(); map.add("files", new FileSystemResource(file.getFile())); map.add("filename", uuid + ".pdf"); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.MULTIPART_FORM_DATA); headers.add("Content-disposition", "attachment; filename=" + uuid + ".pdf"); headers.add("Content-type", "application/pdf"); HttpEntity> requestEntity = new HttpEntity>( map, headers); byte[] queueResult = new RestTemplate().postForObject(environment.getProperty("pdf.converter.url") + "convert/office" , requestEntity, byte[].class); File resultPdf = new File(environment.getProperty("temp.temp") + uuid + ".pdf"); FileOutputStream output = new FileOutputStream(resultPdf); IOUtils.write(queueResult, output); output.close(); Files.deleteIfExists(file.getFile().toPath()); return resultPdf; } private File extractFromZip(File file, String filename) throws IOException { byte[] buffer = new byte[1024]; File newFile = new File(filename); ZipInputStream zis = new ZipInputStream(new FileInputStream(file)); ZipEntry zipEntry = zis.getNextEntry(); while (zipEntry != null) { String zippedFileName = zipEntry.getName(); if (zippedFileName.equals("pdf")) { FileOutputStream fos = new FileOutputStream(newFile); int len; while ((len = zis.read(buffer)) > 0) { fos.write(buffer, 0, len); } fos.close(); zipEntry = zis.getNextEntry(); } } zis.closeEntry(); zis.close(); return newFile; } public eu.eudat.data.entities.Dataset createOrUpdate(DatasetWizardModel datasetWizardModel, Principal principal) throws Exception { Boolean sendNotification = false; Dataset tempDataset = null; DMP dmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(datasetWizardModel.getDmp().getId()); if (datasetWizardModel.getId() != null) { tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(datasetWizardModel.getId()); if (tempDataset != null) { Instant dbTime = Instant.ofEpochMilli(tempDataset.getModified().getTime()).truncatedTo(ChronoUnit.SECONDS); Instant modelTime = Instant.ofEpochMilli(datasetWizardModel.getModified().getTime()).truncatedTo(ChronoUnit.SECONDS); if (modelTime.toEpochMilli() != dbTime.toEpochMilli()) { throw new Exception("Dataset has been modified already by another user."); } sendNotification = true; } } else { MetricsManager.increaseValue("draftDatasets", 1); } if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null) throw new Exception("DMP is finalized, therefore Dataset cannot be edited."); eu.eudat.data.entities.Dataset dataset = datasetWizardModel.toDataModel(); dataset.setDmp(dmp); dataset.setProperties(propertiesModelToString(datasetWizardModel.getDatasetProfileDefinition())); if (this.apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().getClient() != null) { this.getTagsFromProfile(datasetWizardModel, dataset); } if (datasetWizardModel.getStatus() == (int) Dataset.Status.FINALISED.getValue()) checkDatasetValidation(dataset); UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); dataset.setCreator(userInfo); createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), dataset); createExternalDatasetsIfTheyDontExist(dataset); createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), dataset); createServicesIfTheyDontExist(dataset); Dataset dataset1 = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(dataset); datasetWizardModel.setId(dataset1.getId()); if (datasetWizardModel.getDmp().getGrant() == null) { datasetWizardModel.setDmp(new DataManagementPlan().fromDataModelNoDatasets(dataset1.getDmp())); } dataset1.setProfile(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(datasetWizardModel.getProfile().getId())); // datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset1)); updateTags(dataset1, datasetWizardModel.getTags()); if (sendNotification) { if (dataset1.getStatus() != Dataset.Status.FINALISED.getValue()) { this.sendNotification(dataset1, dataset1.getDmp(), userInfo, NotificationType.DATASET_MODIFIED); } else { this.sendNotification(dataset1, dataset1.getDmp(), userInfo, NotificationType.DATASET_MODIFIED_FINALISED); } } return dataset1; } private void sendNotification(Dataset dataset, DMP dmp, UserInfo user, NotificationType notificationType) { List userDMPS = databaseRepository.getUserDmpDao().asQueryable().where(((builder, root) -> builder.equal(root.get("dmp").get("id"), dmp.getId()))).toList(); for (UserDMP userDMP : userDMPS) { if (!userDMP.getUser().getId().equals(user.getId())) { Notification notification = new Notification(); notification.setUserId(user); notification.setType(notificationType); notification.setNotifyState(NotifyState.PENDING); notification.setIsActive(ActiveStatus.ACTIVE); notification.setData("{" + "\"userId\": \"" + userDMP.getUser().getId() + "\"" + ", \"id\": \"" + dataset.getId() + "\"" + ", \"name\": \"" + dataset.getLabel() + "\"" + ", \"path\": \"" + notificationPaths.get(notificationType) + "\"" + "}"); notification.setCreatedAt(new Date()); notification.setUpdatedAt(notification.getCreatedAt()); notification.setContactTypeHint(ContactType.EMAIL); notification.setContactHint(userDMP.getUser().getEmail()); databaseRepository.getNotificationDao().createOrUpdate(notification); } } } private void checkDatasetValidation(Dataset dataset) throws Exception { List datasetProfileValidators = new LinkedList<>(); DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(dataset.getProfile().getId()); DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = builderFactory.newDocumentBuilder(); Document xmlDocument = builder.parse(new ByteArrayInputStream(profile.getDefinition().getBytes())); XPath xPath = XPathFactory.newInstance().newXPath(); String expression = "//validation/@type[.=1]/ancestor::field/@id"; NodeList nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET); for (int i = 0; i < nodeList.getLength(); i++) { Node node = nodeList.item(i); datasetProfileValidators.add(node.getNodeValue()); } JSONObject obj = new JSONObject(dataset.getProperties()); for (String validator : datasetProfileValidators) { if (obj.getString(validator) == null || obj.getString(validator).trim().isEmpty()) { throw new Exception("Field value of " + validator + " must be filled."); } } } private String propertiesModelToString(PagedDatasetProfile pagedDatasetProfile) { Map values = new HashMap(); pagedDatasetProfile.toMap(values); JSONObject jobject = new JSONObject(values); return jobject.toString(); } public void updateTags(Dataset datasetEntity, List tags) throws Exception { // if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) { /*eu.eudat.elastic.entities.Dataset dataset = new eu.eudat.elastic.entities.Dataset(); dataset.setId(datasetWizardModel.getId().toString()); if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) { DatasetCriteria criteria = new DatasetCriteria(); criteria.setTags(datasetWizardModel.getTags()); List tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream) .filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> datasetWizardModel.getTags().stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList()); if (tags.isEmpty()) { datasetWizardModel.getTags().forEach(tag -> tag.setId(UUID.randomUUID().toString())); dataset.setTags(datasetWizardModel.getTags()); } else { dataset.setTags(tags); } } dataset.setLabel(datasetWizardModel.getLabel()); dataset.setDescription(datasetWizardModel.getDescription()); dataset.setTemplate(datasetWizardModel.getProfile()); dataset.setStatus(datasetWizardModel.getStatus()); dataset.setDmp(datasetWizardModel.getDmp().getId()); dataset.setGroup(datasetWizardModel.getDmp().getGroupId()); dataset.setGrant(datasetWizardModel.getDmp().getGrant().getId()); if (datasetWizardModel.getDmp().getUsers() != null) { dataset.setCollaborators(datasetWizardModel.getDmp().getUsers().stream().map(user -> { Collaborator collaborator = new Collaborator(); collaborator.setId(user.getId().toString()); collaborator.setName(user.getName()); return collaborator; }).collect(Collectors.toList())); } DataManagementPlanCriteria dmpCriteria = new DataManagementPlanCriteria(); dmpCriteria.setAllVersions(true); dmpCriteria.setGroupIds(Collections.singletonList(datasetWizardModel.getDmp().getGroupId())); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream() .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> dataset.setLastVersion(dmp.getId().equals(datasetWizardModel.getDmp().getId()))); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream().filter(DMP::isPublic) .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> dataset.setLastPublicVersion(dmp.getId().equals(datasetWizardModel.getDmp().getId()))); if (dataset.getLastVersion() == null) { dataset.setLastVersion(true); } if (dataset.getLastPublicVersion() == null) { dataset.setLastPublicVersion(false); } if (datasetWizardModel.getDmp().getOrganisations() != null) { dataset.setOrganizations(datasetWizardModel.getDmp().getOrganisations().stream().map(org -> { Organization organization = new Organization(); organization.setId(org.getId()); organization.setName(org.getName()); return organization; }).collect(Collectors.toList())); } dataset.setPublic(datasetWizardModel.getDmp().getPublic()); dataset.setGrantStatus(datasetWizardModel.getDmp().getGrant().getStatus()); dataset.setFormData(this.getWordDocumentText(datasetWizardModel));*/ DatasetMapper mapper = new DatasetMapper(apiContext, this); eu.eudat.elastic.entities.Dataset dataset = mapper.toElastic(datasetEntity, tags); apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(dataset); // } } private void createRegistriesIfTheyDontExist(RegistryDao registryDao, eu.eudat.data.entities.Dataset dataset) { if (dataset.getRegistries() != null && !dataset.getRegistries().isEmpty()) { for (eu.eudat.data.entities.Registry registry : dataset.getRegistries()) { RegistryCriteria criteria = new RegistryCriteria(); criteria.setLike(registry.getReference()); List entries = registryDao.getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) registry.setId(entries.get(0).getId()); else { registry.setCreated(new Date()); registryDao.createOrUpdate(registry); } } } } private void createDataRepositoriesIfTheyDontExist(DataRepositoryDao dataRepositoryDao, eu.eudat.data.entities.Dataset dataset) { if (dataset.getDatasetDataRepositories() != null && !dataset.getDatasetDataRepositories().isEmpty()) { for (eu.eudat.data.entities.DatasetDataRepository datasetDataRepository : dataset.getDatasetDataRepositories()) { DataRepositoryCriteria criteria = new DataRepositoryCriteria(); criteria.setLike(datasetDataRepository.getDataRepository().getReference()); List entries = dataRepositoryDao.getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) { datasetDataRepository.getDataRepository().setId(entries.get(0).getId()); datasetDataRepository.setDataset(dataset); dataset.getDatasetDataRepositories().add(datasetDataRepository); } else { datasetDataRepository.getDataRepository().setId(UUID.randomUUID()); DataRepository dataRepository = dataRepositoryDao.createOrUpdate(datasetDataRepository.getDataRepository()); datasetDataRepository.setDataset(dataset); datasetDataRepository.setDataRepository(dataRepository); dataset.getDatasetDataRepositories().add(datasetDataRepository); } } } } private void createServicesIfTheyDontExist(eu.eudat.data.entities.Dataset dataset) { if (dataset.getServices() != null && !dataset.getServices().isEmpty()) { for (DatasetService service : dataset.getServices()) { ServiceCriteria criteria = new ServiceCriteria(); criteria.setLike(service.getService().getReference()); List entries = databaseRepository.getServiceDao().getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) { service.setDataset(dataset); service.getService().setCreated(new Date()); service.setService(service.getService()); this.databaseRepository.getServiceDao().createOrUpdate(service.getService()); dataset.getServices().add(service); } } } } private void createExternalDatasetsIfTheyDontExist(eu.eudat.data.entities.Dataset dataset) { if (dataset.getDatasetExternalDatasets() != null && !dataset.getDatasetExternalDatasets().isEmpty()) { for (eu.eudat.data.entities.DatasetExternalDataset datasetExternalDataset : dataset.getDatasetExternalDatasets()) { ExternalDatasetCriteria criteria = new ExternalDatasetCriteria(); criteria.setLike(datasetExternalDataset.getExternalDataset().getReference()); List entries = databaseRepository.getExternalDatasetDao().getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) { datasetExternalDataset.getExternalDataset().setId(entries.get(0).getId()); datasetExternalDataset.setDataset(dataset); dataset.getDatasetExternalDatasets().add(datasetExternalDataset); } else { datasetExternalDataset.getExternalDataset().setId(UUID.randomUUID()); datasetExternalDataset.setDataset(dataset); ExternalDataset externalDataset = databaseRepository.getExternalDatasetDao().createOrUpdate(datasetExternalDataset.getExternalDataset()); datasetExternalDataset.setExternalDataset(externalDataset); dataset.getDatasetExternalDatasets().add(datasetExternalDataset); } } } } public void makePublic(DatasetDao datasetDao, UUID id) throws Exception { eu.eudat.data.entities.Dataset dataset = datasetDao.find(id); if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue()) throw new Exception("You cannot make public a Dataset That Has not Been Finalised"); datasetDao.createOrUpdate(dataset); MetricsManager.increaseValue("publishedDatasets", 1); } public ResponseEntity getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException { FileEnvelope envelope = getXmlDocument(id, visibilityRuleService, principal); InputStream resource = new FileInputStream(envelope.getFile()); logger.info("Mime Type of " + envelope.getFilename() + " is " + new MimetypesFileTypeMap().getContentType(envelope.getFile())); HttpHeaders responseHeaders = new HttpHeaders(); responseHeaders.setContentLength(envelope.getFile().length()); responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); responseHeaders.set("Content-Disposition", "attachment;filename=" + envelope.getFilename() + ".xml"); responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition"); responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type"); byte[] content = org.apache.poi.util.IOUtils.toByteArray(resource); resource.close(); Files.deleteIfExists(envelope.getFile().toPath()); return new ResponseEntity<>(content, responseHeaders, HttpStatus.OK); } public eu.eudat.data.entities.Dataset createDatasetFromXml(MultipartFile importFile, String dmpId, String datasetProfileId, Principal principal) throws JAXBException, IOException { DatasetImportPagedDatasetProfile importModel = new DatasetImportPagedDatasetProfile(); JAXBContext jaxbContext; // Parses XML into DatasetImport Model. try { InputStream in = importFile.getInputStream(); jaxbContext = JAXBContext.newInstance(DatasetImportPagedDatasetProfile.class); Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); DatasetImportPagedDatasetProfile datasetImport = (DatasetImportPagedDatasetProfile) jaxbUnmarshaller.unmarshal(in); importModel = datasetImport; } catch (IOException e) { logger.error(e.getMessage(), e); } // Checks if XML datasetProfileId GroupId matches the one selected. try { eu.eudat.data.entities.DatasetProfile importDatasetProfile = databaseRepository.getDatasetProfileDao().find(UUID.fromString(importModel.getDatasetProfileId())); eu.eudat.data.entities.DatasetProfile latestVersionDatasetProfile = databaseRepository.getDatasetProfileDao().find(UUID.fromString(datasetProfileId)); if (latestVersionDatasetProfile.getGroupId() != importDatasetProfile.getGroupId()) { throw new Exception(); } } catch (Exception e) { logger.error(e.getMessage(), e); return null; } // Creates the Hash Map to place the values of the data set. Map importMap = importModel.getPages().stream() .flatMap(s -> s.getSections().getSection().stream() .flatMap(cFields -> cFields.getCompositeFields().stream() .flatMap(cField -> cField.getCompositeField().stream() .filter(Objects::nonNull) .flatMap(fields -> fields.getFields().stream() .flatMap(field -> field.getField().stream() .filter(f -> f.getValue() != null) ))))) .collect(Collectors.toMap(DatasetImportField::getId, DatasetImportField::getValue)); // Transforms map into json file. JSONObject jsonDatasetProperties = new JSONObject(importMap); // Creates the entity data set to save. eu.eudat.data.entities.Dataset entity = new Dataset(); entity.setProperties(jsonDatasetProperties.toString()); entity.setLabel(importFile.getOriginalFilename()); DMP dmp = new DMP(); dmp.setId(UUID.fromString(dmpId)); entity.setDmp(dmp); entity.setStatus((short) 0); entity.setCreated(new Date()); entity.setModified(new Date()); DatasetProfile profile = new DatasetProfile(); profile.setId(UUID.fromString(datasetProfileId)); entity.setProfile(profile); UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); entity.setCreator(userInfo); updateTagsXmlImportDataset(apiContext.getOperationsContext().getElasticRepository().getDatasetRepository(), entity); createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), entity); createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), entity); createServicesIfTheyDontExist(entity); createExternalDatasetsIfTheyDontExist(entity); MetricsManager.increaseValue("draftDatasets", 1); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity); } public void updateTagsXmlImportDataset(DatasetRepository datasetRepository, Dataset dataset) throws IOException { // TODO: When tags functionality return. } public DatasetWizardModel datasetUpdateProfile(String id) { DatasetWizardModel dataset = new DatasetWizardModel(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); eu.eudat.elastic.entities.Dataset datasetElastic; try { datasetElastic = datasetRepository.exists() ? datasetRepository.findDocument(id) : new eu.eudat.elastic.entities.Dataset(); } catch (Exception ex) { logger.warn(ex.getMessage()); datasetElastic = null; } dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity)); dataset.fromDataModel(datasetEntity); // Creates the Criteria to get all version of DatasetProfile in question. DatasetProfileCriteria profileCriteria = new DatasetProfileCriteria(); UUID profileId = datasetEntity.getProfile().getGroupId(); List uuidList = new LinkedList<>(); uuidList.add(profileId); profileCriteria.setGroupIds(uuidList); // Gets the latest version of the datasetProfile. eu.eudat.data.entities.DatasetProfile item = databaseRepository.getDatasetProfileDao().getWithCriteria(profileCriteria).getSingle(); // Sets the latest version of dataet Profile to the Dataset in question. dataset.setDatasetProfileDefinition(getLatestDatasetProfile(datasetEntity, item)); dataset.setProfile(new DatasetProfileOverviewModel().fromDataModel(item)); // Now at latest version. dataset.setIsProfileLatestVersion(true); if (datasetElastic != null && datasetElastic.getTags() != null && !datasetElastic.getTags().isEmpty()) { dataset.setTags(datasetElastic.getTags()); } /*if (datasetElastic != null && datasetElastic.getLabel() != null && !datasetElastic.getLabel().isEmpty()) { dataset.setLabel(datasetElastic.getLabel()); }*/ return dataset; } public PagedDatasetProfile getLatestDatasetProfile(Dataset datasetEntity, DatasetProfile profile) { eu.eudat.models.data.user.composite.DatasetProfile datasetprofile = userManager.generateDatasetProfileModel(profile); datasetprofile.setStatus(datasetEntity.getStatus()); if (datasetEntity.getProperties() != null) { JSONObject jobject = new JSONObject(datasetEntity.getProperties()); Map properties = jobject.toMap(); datasetprofile.fromJsonObject(properties); } PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile(); pagedDatasetProfile.buildPagedDatasetProfile(datasetprofile); return pagedDatasetProfile; } public DataTableData getDatasetProfilesUsedByDatasets(DatasetProfileTableRequestItem datasetProfileTableRequestItem, Principal principal) { datasetProfileTableRequestItem.getCriteria().setFilter(DatasetProfileCriteria.DatasetProfileFilter.Datasets.getValue()); datasetProfileTableRequestItem.getCriteria().setUserId(principal.getId()); QueryableList items = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(datasetProfileTableRequestItem.getCriteria()); List listingModels = items.select(item -> new DatasetProfileListingModel().fromDataModel(item)); DataTableData data = new DataTableData<>(); data.setData(listingModels); data.setTotalCount((long) listingModels.size()); return data; } public void generateIndex(Principal principal) { if (principal.getAuthorities().contains(Authorities.ADMIN.getValue())) { this.apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList(); List datasetEntities = new ArrayList<>(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().asQueryable().toList()); datasetEntities.forEach(datasetEntity -> { try { eu.eudat.elastic.entities.Dataset dataset = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(datasetEntity.getId().toString()); updateTags(datasetEntity, dataset != null ? dataset.getTags() : null); } catch (Exception e) { logger.error(e.getMessage(), e); } }); } } public void clearIndex(Principal principal) { if (principal.getAuthorities().contains(Authorities.ADMIN.getValue())) { try { this.apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().clear(); } catch (IOException e) { logger.error(e.getMessage(), e); } } } public void getTagsFromProfile(DatasetWizardModel wizardModel, Dataset dataset) throws IOException { dataset.setProfile(apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(dataset.getProfile().getId())); wizardModel.setDatasetProfileDefinition(this.getPagedProfile(wizardModel, dataset)); ObjectMapper mapper = new ObjectMapper(); String json = mapper.writeValueAsString(wizardModel.getDatasetProfileDefinition()); JsonNode propertiesJson = mapper.readTree(json); List tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().query(new DatasetCriteria()).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream).filter(StreamDistinctBy.distinctByKey(Tag::getId)).collect(Collectors.toList()); Set tagNodes = new HashSet<>(); tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "renderStyle", "tags", true)); tagNodes.addAll(JsonSearcher.findNodes(propertiesJson, "rdaProperty", "dataset.keyword")); if (!tagNodes.isEmpty()) { tagNodes.forEach(node -> { JsonNode value = node.get("value"); String stringValue = value.asText().replaceAll("=", ":"); JSONArray values = new JSONArray(stringValue); if (values != null) { values.iterator().forEachRemaining(element -> { Map data = ((JSONObject) element).toMap(); this.addTag(tags, wizardModel.getTags(), data.get("id").toString(), data.get("name").toString()); }); } else { this.addTag(tags, wizardModel.getTags(), "", value.asText()); } }); } } private void addTag(List srcTags, List dstTags, String id, String name) { Tag tag = new Tag(); if(srcTags.stream().anyMatch(intag -> intag.getName().equals(name))) { tag = srcTags.stream().filter(intag -> intag.getName().equals(name)).findFirst().get(); } else { tag.setName(name); tag.setId(id); } if (dstTags.stream().noneMatch(intag -> intag.getName().equals(name))) { dstTags.add(tag); } } @Transactional private DatasetListingModel mapModel(Dataset item) { if (item.getProfile() == null) return null; DatasetListingModel listingModel = new DatasetListingModel().fromDataModel(item); DatasetProfileCriteria criteria = new DatasetProfileCriteria(); criteria.setGroupIds(Collections.singletonList(item.getProfile().getGroupId())); List profiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList(); boolean islast = false; if (!profiles.isEmpty()) { profiles = profiles.stream().sorted(Comparator.comparing(DatasetProfile::getVersion)).collect(Collectors.toList()); islast = profiles.get(0).getId().equals(item.getProfile().getId()); } listingModel.setProfileLatestVersion(islast); return listingModel; } public long countAllDraft() { eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); criteria.setStatus(0); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); } public long countAllFinalized() { eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); criteria.setStatus(1); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); } public long countAllPublic() { eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); criteria.setIsPublic(true); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); } }