package eu.eudat.logic.managers; import eu.eudat.data.dao.criteria.*; import eu.eudat.data.dao.entities.DataRepositoryDao; import eu.eudat.data.dao.entities.DatasetDao; import eu.eudat.data.dao.entities.RegistryDao; import eu.eudat.data.entities.*; import eu.eudat.data.enumeration.notification.ActiveStatus; import eu.eudat.data.enumeration.notification.ContactType; import eu.eudat.data.enumeration.notification.NotificationType; import eu.eudat.data.enumeration.notification.NotifyState; import eu.eudat.data.query.items.table.dataset.DatasetPublicTableRequest; import eu.eudat.data.query.items.table.dataset.DatasetTableRequest; import eu.eudat.data.query.items.table.datasetprofile.DatasetProfileTableRequestItem; import eu.eudat.elastic.criteria.DatasetCriteria; import eu.eudat.elastic.entities.Collaborator; import eu.eudat.elastic.entities.Organization; import eu.eudat.elastic.repository.DatasetRepository; import eu.eudat.exceptions.security.UnauthorisedException; import eu.eudat.logic.builders.BuilderFactory; import eu.eudat.logic.builders.entity.UserInfoBuilder; import eu.eudat.logic.proxy.config.configloaders.ConfigLoader; import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.services.forms.VisibilityRuleService; import eu.eudat.logic.services.operations.DatabaseRepository; import eu.eudat.logic.utilities.documents.helpers.FileEnvelope; import eu.eudat.logic.utilities.documents.types.ParagraphStyle; import eu.eudat.logic.utilities.documents.word.WordBuilder; import eu.eudat.logic.utilities.documents.xml.ExportXmlBuilder; import eu.eudat.models.HintedModelFactory; import eu.eudat.models.data.datasetImport.DatasetImportField; import eu.eudat.models.data.datasetImport.DatasetImportPagedDatasetProfile; import eu.eudat.models.data.datasetprofile.DatasetProfileListingModel; import eu.eudat.models.data.datasetwizard.DatasetWizardModel; import eu.eudat.models.data.dmp.AssociatedProfile; import eu.eudat.models.data.helpers.common.DataTableData; import eu.eudat.models.data.listingmodels.DatasetListingModel; import eu.eudat.models.data.security.Principal; import eu.eudat.models.data.user.composite.PagedDatasetProfile; import eu.eudat.queryable.QueryableList; import eu.eudat.types.Authorities; import org.apache.commons.io.IOUtils; import org.apache.poi.xwpf.usermodel.XWPFDocument; import org.apache.poi.xwpf.usermodel.XWPFParagraph; import org.apache.poi.xwpf.usermodel.XWPFRun; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.core.io.FileSystemResource; import org.springframework.http.*; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.LinkedMultiValueMap; import org.springframework.web.client.RestTemplate; import org.springframework.web.multipart.MultipartFile; import org.w3c.dom.Document; import org.w3c.dom.Node; import org.w3c.dom.NodeList; import javax.activation.MimetypesFileTypeMap; import javax.persistence.criteria.Join; import javax.persistence.criteria.JoinType; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.xpath.XPath; import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; import java.io.*; import java.math.BigInteger; import java.net.URL; import java.nio.file.Files; import java.nio.file.Paths; import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @Component public class DatasetManager { private static final Logger logger = LoggerFactory.getLogger(DatasetManager.class); private final Map notificationPaths = Stream.of(new Object[][] { {NotificationType.DATASET_MODIFIED, "/datasets/edit"}, {NotificationType.DATASET_MODIFIED_FINALISED, "/datasets/edit"} }).collect(Collectors.toMap(data -> (NotificationType) data[0], data -> (String) data[1])); private ApiContext apiContext; private DatabaseRepository databaseRepository; private DatasetRepository datasetRepository; private BuilderFactory builderFactory; private UserManager userManager; @Autowired public DatasetManager(ApiContext apiContext, UserManager userManager) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.datasetRepository = apiContext.getOperationsContext().getDatasetRepository(); this.builderFactory = apiContext.getOperationsContext().getBuilderFactory(); this.userManager = userManager; } public DataTableData getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception { DatasetCriteria datasetCriteria = new DatasetCriteria(); datasetCriteria.setLabel(datasetTableRequest.getCriteria().getLike()); datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetTemplates()); if (datasetTableRequest.getCriteria().getStatus() != null) { datasetCriteria.setStatus(datasetTableRequest.getCriteria().getStatus().shortValue()); } datasetCriteria.setDmps(datasetTableRequest.getCriteria().getDmpIds()); datasetCriteria.setGroupIds(datasetTableRequest.getCriteria().getGroupIds()); datasetCriteria.setGrants(datasetTableRequest.getCriteria().getGrants()); datasetCriteria.setCollaborators(datasetTableRequest.getCriteria().getCollaborators()); datasetCriteria.setAllowAllVersions(datasetTableRequest.getCriteria().getAllVersions()); datasetCriteria.setOrganiztions(datasetTableRequest.getCriteria().getOrganisations()); datasetCriteria.setTags(datasetTableRequest.getCriteria().getTags()); List datasets; try { datasets = datasetRepository.exists() ? datasetRepository.query(datasetCriteria) : null; } catch (Exception ex) { logger.warn(ex.getMessage()); datasets = null; } UserInfo userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); // QueryableList items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); QueryableList items; if (datasets != null) { if (!datasets.isEmpty()) { items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class)); List finalDatasets = datasets; items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))); } else { items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()})); } } else { items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); } List roles = new LinkedList<>(); if (datasetTableRequest.getCriteria().getRole() != null) roles.add(datasetTableRequest.getCriteria().getRole()); QueryableList authItems = databaseRepository.getDatasetDao().getAuthenticated(items, userInfo, roles); QueryableList pagedItems = PaginationManager.applyPaging(authItems, datasetTableRequest); DataTableData dataTable = new DataTableData<>(); CompletableFuture> itemsFuture = pagedItems. selectAsync(item -> new DatasetListingModel().fromDataModel(item)).whenComplete((resultList, throwable) -> { dataTable.setData(resultList); }); CompletableFuture countFuture = authItems.countAsync().whenComplete((count, throwable) -> { dataTable.setTotalCount(count); }); CompletableFuture.allOf(itemsFuture, countFuture).join(); return dataTable; } public DataTableData getPaged(DatasetPublicTableRequest datasetTableRequest, Principal principal) throws Exception { DatasetCriteria datasetCriteria = new DatasetCriteria(); datasetCriteria.setLabel(datasetTableRequest.getCriteria().getLike()); datasetCriteria.setDatasetTemplates(datasetTableRequest.getCriteria().getDatasetProfile()); datasetCriteria.setDmps(datasetTableRequest.getCriteria().getDmpIds()); datasetCriteria.setGrants(datasetTableRequest.getCriteria().getGrants()); List datasets; try { datasets = datasetRepository.exists() ? datasetRepository.query(datasetCriteria) : new LinkedList<>(); } catch (Exception ex) { logger.warn(ex.getMessage()); datasets = null; } datasetTableRequest.setQuery(databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class))); QueryableList items = datasetTableRequest.applyCriteria(); /*QueryableList items; if (datasets != null) { if (!datasets.isEmpty()) { items = databaseRepository.getDatasetDao().asQueryable().withHint(HintedModelFactory.getHint(DatasetListingModel.class)); List finalDatasets = datasets; items.where((builder, root) -> root.get("id").in(finalDatasets.stream().map(x -> UUID.fromString(x.getId())).collect(Collectors.toList()))); } else items = datasetTableRequest.applyCriteria(); items.where((builder, root) -> root.get("id").in(new UUID[]{UUID.randomUUID()})); } else { items = datasetTableRequest.applyCriteria(); }*/ if (principal.getId() != null && datasetTableRequest.getCriteria().getRole() != null) { items.where((builder, root) -> { Join userJoin = root.join("dmp", JoinType.LEFT).join("users", JoinType.LEFT); return builder.and(builder.equal(userJoin.join("user", JoinType.LEFT).get("id"), principal.getId()), builder.equal(userJoin.get("role"), datasetTableRequest.getCriteria().getRole())); }); } String[] strings = new String[1]; strings[0] = "-dmp:publishedAt|join|"; datasetTableRequest.getOrderings().setFields(strings); QueryableList pagedItems = PaginationManager.applyPaging(items, datasetTableRequest); DataTableData dataTable = new DataTableData<>(); CompletableFuture> itemsFuture = pagedItems. selectAsync(item -> new DatasetListingModel().fromDataModel(item)).whenComplete((resultList, throwable) -> { dataTable.setData(resultList); }); CompletableFuture countFuture = pagedItems.countAsync().whenComplete((count, throwable) -> { dataTable.setTotalCount(count); }); CompletableFuture.allOf(itemsFuture, countFuture).join(); return dataTable; } public DatasetWizardModel getSingle(String id, Principal principal) { DatasetWizardModel dataset = new DatasetWizardModel(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); if (datasetEntity.getDmp().getUsers() .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()) .collect(Collectors.toList()).size() == 0) throw new UnauthorisedException(); eu.eudat.elastic.entities.Dataset datasetElastic; try { datasetElastic = datasetRepository.exists() ? datasetRepository.findDocument(id) : new eu.eudat.elastic.entities.Dataset(); } catch (Exception ex) { logger.warn(ex.getMessage()); datasetElastic = null; } dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity)); dataset.fromDataModel(datasetEntity); // Creates the Criteria to get all version of DatasetProfile in question. DatasetProfileCriteria profileCriteria = new DatasetProfileCriteria(); UUID profileId = datasetEntity.getProfile().getGroupId(); List uuidList = new LinkedList<>(); uuidList.add(profileId); profileCriteria.setGroupIds(uuidList); profileCriteria.setAllVersions(true); List profileVersions = databaseRepository.getDatasetProfileDao().getWithCriteria(profileCriteria) .orderBy(((builder, root) -> builder.desc(root.get("version")))) .toList(); List profileVersionsIncluded = new LinkedList<>(); // Iterate through the versions and remove those that are not included in the DMP of the dataset in question. for (DatasetProfile version : profileVersions) { for (AssociatedProfile p : dataset.getDmp().getProfiles()) { if (version.getId().toString().equals(p.getId().toString())) { profileVersionsIncluded.add(version); } } } // Sort the list with the included Versions. Stream sorted = profileVersionsIncluded.stream().sorted(Comparator.comparing(DatasetProfile::getVersion).reversed()); // Make the Stream into List and get the first item. DatasetProfile profile = sorted.collect(Collectors.toList()).iterator().next(); // Check if the dataset is on the latest Version. boolean latestVersion = profile.getVersion().toString().equals(datasetEntity.getProfile().getVersion().toString()); dataset.setIsProfileLatestVersion(latestVersion); if (datasetElastic != null && datasetElastic.getTags() != null && !datasetElastic.getTags().isEmpty()) { dataset.setTags(datasetElastic.getTags()); } /*if (datasetElastic != null && datasetElastic.getLabel() != null && !datasetElastic.getLabel().isEmpty()) { dataset.setLabel(datasetElastic.getLabel()); }*/ return dataset; } public DatasetWizardModel getSinglePublic(String id) throws Exception { DatasetWizardModel dataset = new DatasetWizardModel(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().isPublicDataset(UUID.fromString(id)); if (datasetEntity != null && datasetEntity.getStatus() == 1 && datasetEntity.getDmp().getStatus() == 1) { dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity)); dataset.fromDataModel(datasetEntity); return dataset; } else { throw new Exception("Selected dataset is not public"); } } public PagedDatasetProfile getPagedProfile(DatasetWizardModel dataset, eu.eudat.data.entities.Dataset datasetEntity) { eu.eudat.models.data.user.composite.DatasetProfile datasetprofile = userManager.generateDatasetProfileModel(datasetEntity.getProfile()); datasetprofile.setStatus(dataset.getStatus()); if (datasetEntity.getProperties() != null) { JSONObject jObject = new JSONObject(datasetEntity.getProperties()); Map properties = jObject.toMap(); datasetprofile.fromJsonObject(properties); } PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile(); pagedDatasetProfile.buildPagedDatasetProfile(datasetprofile); return pagedDatasetProfile; } public File getWordDocument(ConfigLoader configLoader, String id, VisibilityRuleService visibilityRuleService, Principal principal) throws IOException { WordBuilder wordBuilder = new WordBuilder(); DatasetWizardModel dataset = new DatasetWizardModel(); XWPFDocument document = configLoader.getDocument(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers() .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()) .collect(Collectors.toList()).size() == 0) throw new UnauthorisedException(); wordBuilder.addParagraphContent(datasetEntity.getLabel(), document, ParagraphStyle.HEADER1, BigInteger.ZERO); // Space below Dataset title. XWPFParagraph parBreakDataset = document.createParagraph(); XWPFParagraph datasetTemplateParagraph = document.createParagraph(); datasetTemplateParagraph.setStyle("Heading2"); XWPFRun runDatasetTemplate1 = datasetTemplateParagraph.createRun(); runDatasetTemplate1.setText("Template: "); runDatasetTemplate1.setBold(true); runDatasetTemplate1.setFontSize(12); XWPFRun runDatasetTemplate = datasetTemplateParagraph.createRun(); runDatasetTemplate.setText(datasetEntity.getProfile().getLabel()); runDatasetTemplate.setColor("2E75B6"); runDatasetTemplate.setBold(true); runDatasetTemplate.setFontSize(12); XWPFParagraph externalReferencesParagraph = document.createParagraph(); externalReferencesParagraph.setStyle("Heading2"); XWPFRun externalReferencesRun = externalReferencesParagraph.createRun(); externalReferencesRun.setText("External References"); externalReferencesRun.setColor("2E75B6"); externalReferencesRun.setBold(true); externalReferencesRun.setFontSize(12); wordBuilder.addParagraphContent("Data Repositories", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.getDatasetDataRepositories().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getDatasetDataRepositories().stream().map(DatasetDataRepository::getDataRepository).map(DataRepository::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.TEXT, BigInteger.ZERO); } wordBuilder.addParagraphContent("External Datasets", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.getDatasetExternalDatasets().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getDatasetExternalDatasets().stream().map(DatasetExternalDataset::getExternalDataset).map(ExternalDataset::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.TEXT, BigInteger.ZERO); } wordBuilder.addParagraphContent("Registries", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.getRegistries().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getRegistries().stream().map(Registry::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.TEXT, BigInteger.ZERO); } wordBuilder.addParagraphContent("Services", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.getServices().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getServices().stream().map(DatasetService::getService).map(Service::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.TEXT, BigInteger.ZERO); } /*wordBuilder.addParagraphContent("Tags", document, ParagraphStyle.HEADER3, BigInteger.ZERO); if (datasetEntity.().size() > 0) { wordBuilder.addParagraphContent(datasetEntity.getServices().stream().map(DatasetService::getService).map(Service::getLabel).collect(Collectors.joining(", ")) , document, ParagraphStyle.HEADER4, BigInteger.ZERO); }*/ Map properties = new HashMap<>(); if (datasetEntity.getProperties() != null) { JSONObject jObject = new JSONObject(datasetEntity.getProperties()); properties = jObject.toMap(); } wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO); PagedDatasetProfile pagedDatasetProfile = getPagedProfile(dataset, datasetEntity); visibilityRuleService.setProperties(properties); visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules()); wordBuilder.build(document, pagedDatasetProfile, visibilityRuleService); String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", ""); File exportFile = new File(label + ".docx"); // Removes the top empty headings. for (int i = 0; i < 6; i++) { document.removeBodyElement(0); } FileOutputStream out = new FileOutputStream(exportFile); document.write(out); out.close(); return exportFile; } public FileEnvelope getXmlDocument(String id, VisibilityRuleService visibilityRuleService, Principal principal) throws InstantiationException, IllegalAccessException, IOException { ExportXmlBuilder xmlBuilder = new ExportXmlBuilder(); DatasetWizardModel dataset = new DatasetWizardModel(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers() .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()) .collect(Collectors.toList()).size() == 0) throw new UnauthorisedException(); Map properties = new HashMap<>(); if (datasetEntity.getProperties() != null) { JSONObject jobject = new JSONObject(datasetEntity.getProperties()); properties = jobject.toMap(); } PagedDatasetProfile pagedDatasetProfile = getPagedProfile(dataset, datasetEntity); visibilityRuleService.setProperties(properties); visibilityRuleService.buildVisibilityContext(pagedDatasetProfile.getRules()); File file = xmlBuilder.build(pagedDatasetProfile, datasetEntity.getProfile().getId(), visibilityRuleService); FileEnvelope fileEnvelope = new FileEnvelope(); fileEnvelope.setFile(file); fileEnvelope.setFilename(datasetEntity.getLabel()); return fileEnvelope; } public File convertToPDF(File file, Environment environment, String label) throws IOException, InterruptedException { LinkedMultiValueMap map = new LinkedMultiValueMap<>(); map.add("files", new FileSystemResource(file)); map.add("filename", label + ".pdf"); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.MULTIPART_FORM_DATA); headers.add("Content-disposition", "attachment; filename=" + label + ".pdf"); headers.add("Content-type", "application/pdf"); HttpEntity> requestEntity = new HttpEntity>( map, headers); byte[] queueResult = new RestTemplate().postForObject(environment.getProperty("pdf.converter.url") + "convert/office" , requestEntity, byte[].class); File resultPdf = new File(label + ".pdf"); FileOutputStream output = new FileOutputStream(resultPdf); IOUtils.write(queueResult, output); output.close(); Files.deleteIfExists(file.toPath()); return resultPdf; } private File extractFromZip(File file, String filename) throws IOException { byte[] buffer = new byte[1024]; File newFile = new File(filename); ZipInputStream zis = new ZipInputStream(new FileInputStream(file)); ZipEntry zipEntry = zis.getNextEntry(); while (zipEntry != null) { String zippedFileName = zipEntry.getName(); if (zippedFileName.equals("pdf")) { FileOutputStream fos = new FileOutputStream(newFile); int len; while ((len = zis.read(buffer)) > 0) { fos.write(buffer, 0, len); } fos.close(); zipEntry = zis.getNextEntry(); } } zis.closeEntry(); zis.close(); return newFile; } public eu.eudat.data.entities.Dataset createOrUpdate(DatasetWizardModel datasetWizardModel, Principal principal) throws Exception { Boolean sendNotification = false; DMP dmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(datasetWizardModel.getDmp().getId()); if (datasetWizardModel.getId() != null) { Dataset tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(datasetWizardModel.getId()); if (tempDataset != null) { if (datasetWizardModel.getModified().getTime() != tempDataset.getModified().getTime()) { throw new Exception("Dataset has been modified already by another user."); } sendNotification = true; } } if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null) throw new Exception("DMP is finalized, therefore Dataset cannot be edited."); eu.eudat.data.entities.Dataset dataset = datasetWizardModel.toDataModel(); dataset.setDmp(dmp); propertiesModelToString(datasetWizardModel, dataset); if (datasetWizardModel.getStatus() == (int) Dataset.Status.FINALISED.getValue()) checkDatasetValidation(dataset); UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); dataset.setCreator(userInfo); createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), dataset); createExternalDatasetsIfTheyDontExist(dataset); createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), dataset); createServicesIfTheyDontExist(dataset); Dataset dataset1 = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(dataset); datasetWizardModel.setId(dataset1.getId()); updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel); if (sendNotification) { if (dataset1.getStatus() != Dataset.Status.FINALISED.getValue()) { this.sendNotification(dataset1, dataset1.getDmp(), userInfo, NotificationType.DATASET_MODIFIED); } else { this.sendNotification(dataset1, dataset1.getDmp(), userInfo, NotificationType.DATASET_MODIFIED_FINALISED); } } return dataset1; } private void sendNotification(Dataset dataset, DMP dmp, UserInfo user, NotificationType notificationType) { List userDMPS = databaseRepository.getUserDmpDao().asQueryable().where(((builder, root) -> builder.equal(root.get("dmp").get("id"), dmp.getId()))).toList(); for (UserDMP userDMP : userDMPS) { if (!userDMP.getUser().getId().equals(user.getId())) { Notification notification = new Notification(); notification.setUserId(user); notification.setType(notificationType); notification.setNotifyState(NotifyState.PENDING); notification.setIsActive(ActiveStatus.ACTIVE); notification.setData("{" + "\"userId\": \"" + userDMP.getUser().getId() + "\"" + ", \"id\": \"" + dataset.getId() + "\"" + ", \"name\": \"" + dataset.getLabel() + "\"" + ", \"path\": \"" + notificationPaths.get(notificationType) + "\"" + "}"); notification.setCreatedAt(new Date()); notification.setUpdatedAt(notification.getCreatedAt()); notification.setContactTypeHint(ContactType.EMAIL); notification.setContactHint(userDMP.getUser().getEmail()); databaseRepository.getNotificationDao().createOrUpdate(notification); } } } private void checkDatasetValidation(Dataset dataset) throws Exception { List datasetProfileValidators = new LinkedList<>(); DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(dataset.getProfile().getId()); DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = builderFactory.newDocumentBuilder(); Document xmlDocument = builder.parse(new ByteArrayInputStream(profile.getDefinition().getBytes())); XPath xPath = XPathFactory.newInstance().newXPath(); String expression = "//validation/@type[.=1]/ancestor::field/@id"; NodeList nodeList = (NodeList) xPath.compile(expression).evaluate(xmlDocument, XPathConstants.NODESET); for (int i = 0; i < nodeList.getLength(); i++) { Node node = nodeList.item(i); datasetProfileValidators.add(node.getNodeValue()); } JSONObject obj = new JSONObject(dataset.getProperties()); for (String validator : datasetProfileValidators) { if (obj.getString(validator) == null || obj.getString(validator).trim().isEmpty()) { throw new Exception("Field value of " + validator + " must be filled."); } } } private void propertiesModelToString(DatasetWizardModel datasetWizardModel, eu.eudat.data.entities.Dataset dataset) { Map values = new HashMap(); PagedDatasetProfile properties = datasetWizardModel.getDatasetProfileDefinition(); properties.toMap(values); JSONObject jobject = new JSONObject(values); dataset.setProperties(jobject.toString()); } public void updateTags(DatasetRepository datasetRepository, DatasetWizardModel datasetWizardModel) throws IOException { // if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) { eu.eudat.elastic.entities.Dataset dataset = new eu.eudat.elastic.entities.Dataset(); dataset.setId(datasetWizardModel.getId().toString()); dataset.setTags(datasetWizardModel.getTags()); dataset.setLabel(datasetWizardModel.getLabel()); dataset.setTemplate(datasetWizardModel.getProfile()); dataset.setStatus(datasetWizardModel.getStatus()); dataset.setDmp(datasetWizardModel.getDmp().getId()); dataset.setGroup(datasetWizardModel.getDmp().getGroupId()); dataset.setGrant(datasetWizardModel.getDmp().getGrant().getId()); dataset.setCollaborators(datasetWizardModel.getDmp().getUsers().stream().map(user -> { Collaborator collaborator = new Collaborator(); collaborator.setId(user.getId().toString()); collaborator.setName(user.getName()); return collaborator; }).collect(Collectors.toList())); DataManagementPlanCriteria dmpCriteria = new DataManagementPlanCriteria(); dmpCriteria.setAllVersions(true); dmpCriteria.setGroupIds(Collections.singletonList(datasetWizardModel.getDmp().getGroupId())); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream() .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> dataset.setLastVersion(dmp.getId().equals(datasetWizardModel.getDmp().getId()))); if (dataset.getLastVersion() == null) { dataset.setLastVersion(true); } dataset.setOrganizations(datasetWizardModel.getDmp().getOrganisations().stream().map(org -> { Organization organization = new Organization(); organization.setId(org.getId()); organization.setName(org.getName()); return organization; }).collect(Collectors.toList())); datasetRepository.createOrUpdate(dataset); // } } private void createRegistriesIfTheyDontExist(RegistryDao registryDao, eu.eudat.data.entities.Dataset dataset) { if (dataset.getRegistries() != null && !dataset.getRegistries().isEmpty()) { for (eu.eudat.data.entities.Registry registry : dataset.getRegistries()) { RegistryCriteria criteria = new RegistryCriteria(); criteria.setLike(registry.getReference()); List entries = registryDao.getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) registry.setId(entries.get(0).getId()); else { registry.setCreated(new Date()); registryDao.createOrUpdate(registry); } } } } private void createDataRepositoriesIfTheyDontExist(DataRepositoryDao dataRepositoryDao, eu.eudat.data.entities.Dataset dataset) { if (dataset.getDatasetDataRepositories() != null && !dataset.getDatasetDataRepositories().isEmpty()) { for (eu.eudat.data.entities.DatasetDataRepository datasetDataRepository : dataset.getDatasetDataRepositories()) { DataRepositoryCriteria criteria = new DataRepositoryCriteria(); criteria.setLike(datasetDataRepository.getDataRepository().getReference()); List entries = dataRepositoryDao.getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) { datasetDataRepository.getDataRepository().setId(entries.get(0).getId()); datasetDataRepository.setDataset(dataset); dataset.getDatasetDataRepositories().add(datasetDataRepository); } else { datasetDataRepository.getDataRepository().setId(UUID.randomUUID()); DataRepository dataRepository = dataRepositoryDao.createOrUpdate(datasetDataRepository.getDataRepository()); datasetDataRepository.setDataset(dataset); datasetDataRepository.setDataRepository(dataRepository); dataset.getDatasetDataRepositories().add(datasetDataRepository); } } } } private void createServicesIfTheyDontExist(eu.eudat.data.entities.Dataset dataset) { if (dataset.getServices() != null && !dataset.getServices().isEmpty()) { for (DatasetService service : dataset.getServices()) { ServiceCriteria criteria = new ServiceCriteria(); criteria.setLike(service.getService().getReference()); List entries = databaseRepository.getServiceDao().getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) { service.setDataset(dataset); service.getService().setCreated(new Date()); service.setService(service.getService()); this.databaseRepository.getServiceDao().createOrUpdate(service.getService()); dataset.getServices().add(service); } } } } private void createExternalDatasetsIfTheyDontExist(eu.eudat.data.entities.Dataset dataset) { if (dataset.getDatasetExternalDatasets() != null && !dataset.getDatasetExternalDatasets().isEmpty()) { for (eu.eudat.data.entities.DatasetExternalDataset datasetExternalDataset : dataset.getDatasetExternalDatasets()) { ExternalDatasetCriteria criteria = new ExternalDatasetCriteria(); criteria.setLike(datasetExternalDataset.getExternalDataset().getReference()); List entries = databaseRepository.getExternalDatasetDao().getWithCriteria(criteria).toList(); if (entries != null && !entries.isEmpty()) { datasetExternalDataset.getExternalDataset().setId(entries.get(0).getId()); datasetExternalDataset.setDataset(dataset); dataset.getDatasetExternalDatasets().add(datasetExternalDataset); } else { datasetExternalDataset.getExternalDataset().setId(UUID.randomUUID()); datasetExternalDataset.setDataset(dataset); ExternalDataset externalDataset = databaseRepository.getExternalDatasetDao().createOrUpdate(datasetExternalDataset.getExternalDataset()); datasetExternalDataset.setExternalDataset(externalDataset); dataset.getDatasetExternalDatasets().add(datasetExternalDataset); } } } } public void makePublic(DatasetDao datasetDao, UUID id) throws Exception { eu.eudat.data.entities.Dataset dataset = datasetDao.find(id); if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue()) throw new Exception("You cannot make public a Dataset That Has not Been Finalised"); datasetDao.createOrUpdate(dataset); } public ResponseEntity getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException { FileEnvelope envelope = getXmlDocument(id, visibilityRuleService, principal); InputStream resource = new FileInputStream(envelope.getFile()); logger.info("Mime Type of " + envelope.getFilename() + " is " + new MimetypesFileTypeMap().getContentType(envelope.getFile())); HttpHeaders responseHeaders = new HttpHeaders(); responseHeaders.setContentLength(envelope.getFile().length()); responseHeaders.setContentType(MediaType.APPLICATION_OCTET_STREAM); responseHeaders.set("Content-Disposition", "attachment;filename=" + envelope.getFilename() + ".xml"); responseHeaders.set("Access-Control-Expose-Headers", "Content-Disposition"); responseHeaders.get("Access-Control-Expose-Headers").add("Content-Type"); byte[] content = org.apache.poi.util.IOUtils.toByteArray(resource); resource.close(); Files.deleteIfExists(envelope.getFile().toPath()); return new ResponseEntity<>(content, responseHeaders, HttpStatus.OK); } public eu.eudat.data.entities.Dataset createDatasetFromXml(MultipartFile importFile, String dmpId, String datasetProfileId, Principal principal) throws JAXBException, IOException { DatasetImportPagedDatasetProfile importModel = new DatasetImportPagedDatasetProfile(); JAXBContext jaxbContext; // Parses XML into DatasetImport Model. try { InputStream in = importFile.getInputStream(); jaxbContext = JAXBContext.newInstance(DatasetImportPagedDatasetProfile.class); Unmarshaller jaxbUnmarshaller = jaxbContext.createUnmarshaller(); DatasetImportPagedDatasetProfile datasetImport = (DatasetImportPagedDatasetProfile) jaxbUnmarshaller.unmarshal(in); importModel = datasetImport; } catch (IOException e) { logger.error(e.getMessage(), e); } // Checks if XML datasetProfileId GroupId matches the one selected. try { eu.eudat.data.entities.DatasetProfile importDatasetProfile = databaseRepository.getDatasetProfileDao().find(UUID.fromString(importModel.getDatasetProfileId())); eu.eudat.data.entities.DatasetProfile latestVersionDatasetProfile = databaseRepository.getDatasetProfileDao().find(UUID.fromString(datasetProfileId)); if (latestVersionDatasetProfile.getGroupId() != importDatasetProfile.getGroupId()) { throw new Exception(); } } catch (Exception e) { logger.error(e.getMessage(), e); return null; } // Creates the Hash Map to place the values of the data set. Map importMap = importModel.getPages().stream() .flatMap(s -> s.getSections().getSection().stream() .flatMap(cFields -> cFields.getCompositeFields().stream() .flatMap(cField -> cField.getCompositeField().stream() .filter(Objects::nonNull) .flatMap(fields -> fields.getFields().stream() .flatMap(field -> field.getField().stream() .filter(f -> f.getValue() != null) ))))) .collect(Collectors.toMap(DatasetImportField::getId, DatasetImportField::getValue)); // Transforms map into json file. JSONObject jsonDatasetProperties = new JSONObject(importMap); // Creates the entity data set to save. eu.eudat.data.entities.Dataset entity = new Dataset(); entity.setProperties(jsonDatasetProperties.toString()); entity.setLabel(importFile.getOriginalFilename()); DMP dmp = new DMP(); dmp.setId(UUID.fromString(dmpId)); entity.setDmp(dmp); entity.setStatus((short) 0); entity.setCreated(new Date()); entity.setModified(new Date()); DatasetProfile profile = new DatasetProfile(); profile.setId(UUID.fromString(datasetProfileId)); entity.setProfile(profile); UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); entity.setCreator(userInfo); updateTagsXmlImportDataset(apiContext.getOperationsContext().getDatasetRepository(), entity); createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), entity); createDataRepositoriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getDataRepositoryDao(), entity); createServicesIfTheyDontExist(entity); createExternalDatasetsIfTheyDontExist(entity); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity); } public void updateTagsXmlImportDataset(DatasetRepository datasetRepository, Dataset dataset) throws IOException { // TODO: When tags functionality return. } public DatasetWizardModel datasetUpdateProfile(String id) { DatasetWizardModel dataset = new DatasetWizardModel(); eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); eu.eudat.elastic.entities.Dataset datasetElastic; try { datasetElastic = datasetRepository.exists() ? datasetRepository.findDocument(id) : new eu.eudat.elastic.entities.Dataset(); } catch (Exception ex) { logger.warn(ex.getMessage()); datasetElastic = null; } dataset.setDatasetProfileDefinition(getPagedProfile(dataset, datasetEntity)); dataset.fromDataModel(datasetEntity); // Creates the Criteria to get all version of DatasetProfile in question. DatasetProfileCriteria profileCriteria = new DatasetProfileCriteria(); UUID profileId = datasetEntity.getProfile().getGroupId(); List uuidList = new LinkedList<>(); uuidList.add(profileId); profileCriteria.setGroupIds(uuidList); // Gets the latest version of the datasetProfile. eu.eudat.data.entities.DatasetProfile item = databaseRepository.getDatasetProfileDao().getWithCriteria(profileCriteria).getSingle(); // Sets the latest version of dataet Profile to the Dataset in question. dataset.setDatasetProfileDefinition(getLatestDatasetProfile(datasetEntity, item)); dataset.setProfile(item.getId()); // Now at latest version. dataset.setIsProfileLatestVersion(true); if (datasetElastic != null && datasetElastic.getTags() != null && !datasetElastic.getTags().isEmpty()) { dataset.setTags(datasetElastic.getTags()); } /*if (datasetElastic != null && datasetElastic.getLabel() != null && !datasetElastic.getLabel().isEmpty()) { dataset.setLabel(datasetElastic.getLabel()); }*/ return dataset; } public PagedDatasetProfile getLatestDatasetProfile(Dataset datasetEntity, DatasetProfile profile) { eu.eudat.models.data.user.composite.DatasetProfile datasetprofile = userManager.generateDatasetProfileModel(profile); datasetprofile.setStatus(datasetEntity.getStatus()); if (datasetEntity.getProperties() != null) { JSONObject jobject = new JSONObject(datasetEntity.getProperties()); Map properties = jobject.toMap(); datasetprofile.fromJsonObject(properties); } PagedDatasetProfile pagedDatasetProfile = new PagedDatasetProfile(); pagedDatasetProfile.buildPagedDatasetProfile(datasetprofile); return pagedDatasetProfile; } public DataTableData getDatasetProfilesUsedByDatasets(DatasetProfileTableRequestItem datasetProfileTableRequestItem, Principal principal) { datasetProfileTableRequestItem.getCriteria().setFilter(DatasetProfileCriteria.DatasetProfileFilter.Datasets.getValue()); datasetProfileTableRequestItem.getCriteria().setUserId(principal.getId()); QueryableList items = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(datasetProfileTableRequestItem.getCriteria()); List listingModels = items.select(item -> new DatasetProfileListingModel().fromDataModel(item)); DataTableData data = new DataTableData<>(); data.setData(listingModels); data.setTotalCount((long) listingModels.size()); return data; } }