From aa1156f940f6be851fa59e2edf2d533fb5fbe370 Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Wed, 18 Mar 2020 18:23:34 +0200 Subject: [PATCH] Add Form Data index for dataset --- .../eu/eudat/elastic/entities/Dataset.java | 11 ++ .../elastic/repository/DatasetRepository.java | 3 +- .../controllers/DatasetWizardController.java | 4 +- .../managers/DataManagementPlanManager.java | 15 ++- .../eudat/logic/managers/DatasetManager.java | 124 ++++++++++++++---- 5 files changed, 121 insertions(+), 36 deletions(-) diff --git a/dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java b/dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java index d4a205f94..ee538c06b 100644 --- a/dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java +++ b/dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java @@ -58,6 +58,7 @@ public class Dataset implements ElasticEntity { private List organizations; private Boolean isPublic; private Short grantStatus; + private String formData; public String getId() { return id; @@ -179,6 +180,14 @@ public class Dataset implements ElasticEntity { this.grantStatus = grantStatus; } + public String getFormData() { + return formData; + } + + public void setFormData(String formData) { + this.formData = formData; + } + @Override public XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException { builder.startObject(); @@ -235,6 +244,7 @@ public class Dataset implements ElasticEntity { if (this.grantStatus != null) { builder.field("grantStatus", this.grantStatus.toString()); } + builder.field("formData", this.formData); builder.endObject(); return builder; } @@ -267,6 +277,7 @@ public class Dataset implements ElasticEntity { if (fields.get("grantStatus") != null) { this.grantStatus = Short.valueOf((String) fields.get("grantStatus")); } + this.formData = (String) fields.get("formData"); } return this; } diff --git a/dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java b/dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java index aedde08bc..6e862df5f 100644 --- a/dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java +++ b/dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java @@ -71,9 +71,10 @@ public class DatasetRepository extends ElasticRepository (String) data[0], data -> (Float) data[1])))); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/DatasetWizardController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/DatasetWizardController.java index b5967aa67..bed8c804a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/DatasetWizardController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/DatasetWizardController.java @@ -88,7 +88,7 @@ public class DatasetWizardController extends BaseController { VisibilityRuleService visibilityRuleService = this.getApiContext().getUtilitiesService().getVisibilityRuleService(); return this.datasetManager.getDocument(id, visibilityRuleService, contentType, principal); } else if (contentType.equals("application/msword")) { - File file = datasetManager.getWordDocument(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal); + File file = datasetManager.getWordDocumentFile(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal); InputStream resource = new FileInputStream(file); HttpHeaders responseHeaders = new HttpHeaders(); responseHeaders.setContentLength(file.length()); @@ -152,7 +152,7 @@ public class DatasetWizardController extends BaseController { @RequestMapping(method = RequestMethod.GET, value = {"/getPDF/{id}"}) public @ResponseBody ResponseEntity getPDFDocument(@PathVariable String id, Principal principal) throws IllegalAccessException, IOException, InstantiationException, InterruptedException { - File file = datasetManager.getWordDocument(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal); + File file = datasetManager.getWordDocumentFile(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal); String fileName = file.getName(); if (fileName.endsWith(".docx")){ fileName = fileName.substring(0, fileName.length() - 5); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java index 361f508ad..7711314f5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java @@ -733,8 +733,9 @@ public class DataManagementPlanManager { try { DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset); datasetWizardModel.setTags(apiContext.getOperationsContext().getDatasetRepository().findDocument(dataset.getId().toString()).getTags()); + datasetWizardModel.setDatasetProfileDefinition(this.datasetManager.getPagedProfile(datasetWizardModel, dataset)); this.datasetManager.updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel); - } catch (IOException e) { + } catch (Exception e) { logger.error(e.getMessage(), e); } } @@ -884,8 +885,9 @@ public class DataManagementPlanManager { try { DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(entityDataset); datasetWizardModel.setTags(apiContext.getOperationsContext().getDatasetRepository().findDocument(dataset.getId().toString()).getTags()); + datasetWizardModel.setDatasetProfileDefinition(this.datasetManager.getPagedProfile(datasetWizardModel, entityDataset)); this.datasetManager.updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel); - } catch (IOException e) { + } catch (Exception e) { logger.error(e.getMessage(), e); } newDataset.update(entityDataset); @@ -971,8 +973,12 @@ public class DataManagementPlanManager { try { datasetElastic.setTags(apiContext.getOperationsContext().getDatasetRepository().findDocument(dataset.getId().toString()).getTags()); + DatasetWizardModel datasetWizardModel = new DatasetWizardModel(); + datasetWizardModel.setStatus(dataset1.getStatus()); + datasetWizardModel.setDatasetProfileDefinition(this.datasetManager.getPagedProfile(datasetWizardModel, dataset1)); + datasetElastic.setFormData(this.datasetManager.getWordDocumentText(datasetWizardModel)); apiContext.getOperationsContext().getDatasetRepository().createOrUpdate(datasetElastic); - } catch (IOException e) { + } catch (Exception e) { logger.error(e.getMessage(), e); } }))); @@ -1343,8 +1349,9 @@ public class DataManagementPlanManager { try { DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset); datasetWizardModel.setTags(apiContext.getOperationsContext().getDatasetRepository().findDocument(dataset.getId().toString()).getTags()); + datasetWizardModel.setDatasetProfileDefinition(this.datasetManager.getPagedProfile(datasetWizardModel, dataset)); this.datasetManager.updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel); - } catch (IOException e) { + } catch (Exception e) { logger.error(e.getMessage(), e); } }); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java index ce0744cc6..94d245263 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java @@ -43,6 +43,7 @@ import eu.eudat.models.data.user.composite.PagedDatasetProfile; import eu.eudat.queryable.QueryableList; import eu.eudat.types.Authorities; import org.apache.commons.io.IOUtils; +import org.apache.poi.xwpf.extractor.XWPFWordExtractor; import org.apache.poi.xwpf.usermodel.XWPFDocument; import org.apache.poi.xwpf.usermodel.XWPFParagraph; import org.apache.poi.xwpf.usermodel.XWPFRun; @@ -64,6 +65,7 @@ import org.w3c.dom.NodeList; import javax.activation.MimetypesFileTypeMap; import javax.persistence.criteria.Join; import javax.persistence.criteria.JoinType; +import javax.transaction.Transactional; import javax.xml.bind.JAXBContext; import javax.xml.bind.JAXBException; import javax.xml.bind.Unmarshaller; @@ -96,14 +98,16 @@ public class DatasetManager { private DatasetRepository datasetRepository; private BuilderFactory builderFactory; private UserManager userManager; + private ConfigLoader configLoader; @Autowired - public DatasetManager(ApiContext apiContext, UserManager userManager) { + public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.datasetRepository = apiContext.getOperationsContext().getDatasetRepository(); this.builderFactory = apiContext.getOperationsContext().getBuilderFactory(); this.userManager = userManager; + this.configLoader = configLoader; } public DataTableData getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception { @@ -331,15 +335,11 @@ public class DatasetManager { return pagedDatasetProfile; } - public File getWordDocument(ConfigLoader configLoader, String id, VisibilityRuleService visibilityRuleService, Principal principal) throws IOException { + private XWPFDocument getWordDocument(ConfigLoader configLoader, eu.eudat.data.entities.Dataset datasetEntity, VisibilityRuleService visibilityRuleService) throws IOException { WordBuilder wordBuilder = new WordBuilder(); DatasetWizardModel dataset = new DatasetWizardModel(); XWPFDocument document = configLoader.getDocument(); - eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); - if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers() - .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()) - .collect(Collectors.toList()).size() == 0) - throw new UnauthorisedException(); + wordBuilder.addParagraphContent(datasetEntity.getLabel(), document, ParagraphStyle.HEADER1, BigInteger.ZERO); // Space below Dataset title. @@ -410,10 +410,62 @@ public class DatasetManager { document.removeBodyElement(0); } + return document; + //FileOutputStream out = new FileOutputStream(exportFile); + // document.write(out); + // out.close(); + // return exportFile; + } + + private XWPFDocument getightWordDocument(ConfigLoader configLoader, DatasetWizardModel dataset, VisibilityRuleService visibilityRuleService) throws IOException { + WordBuilder wordBuilder = new WordBuilder(); + XWPFDocument document = configLoader.getDocument(); + + // Space below Dataset title. + XWPFParagraph parBreakDataset = document.createParagraph(); + + Map properties = new HashMap<>(); + if (dataset.getDatasetProfileDefinition() != null) { + JSONObject jObject = new JSONObject(propertiesModelToString(dataset.getDatasetProfileDefinition())); + properties = jObject.toMap(); + } + + wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO); + visibilityRuleService.setProperties(properties); + visibilityRuleService.buildVisibilityContext(dataset.getDatasetProfileDefinition().getRules()); + wordBuilder.build(document, dataset.getDatasetProfileDefinition(), visibilityRuleService); + + // Removes the top empty headings. + for (int i = 0; i < 6; i++) { + document.removeBodyElement(0); + } + + return document; + //FileOutputStream out = new FileOutputStream(exportFile); + // document.write(out); + // out.close(); + // return exportFile; + } + + public File getWordDocumentFile(ConfigLoader configLoader, String id, VisibilityRuleService visibilityRuleService, Principal principal) throws IOException { + eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class)); + if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers() + .stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId()) + .collect(Collectors.toList()).size() == 0) + throw new UnauthorisedException(); + String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", ""); + File exportFile = new File(label + ".docx"); + XWPFDocument document = getWordDocument(configLoader, datasetEntity, visibilityRuleService); FileOutputStream out = new FileOutputStream(exportFile); - document.write(out); - out.close(); - return exportFile; + document.write(out); + out.close(); + return exportFile; + } + + public String getWordDocumentText (DatasetWizardModel datasetEntity) throws Exception { + XWPFDocument document = getightWordDocument(this.configLoader, datasetEntity, this.apiContext.getUtilitiesService().getVisibilityRuleService()); + XWPFWordExtractor extractor = new XWPFWordExtractor(document); + return extractor.getText();/*.replaceAll("\n\\s*", " ");*/ } public FileEnvelope getXmlDocument(String id, VisibilityRuleService visibilityRuleService, Principal principal) throws InstantiationException, IllegalAccessException, IOException { @@ -488,9 +540,10 @@ public class DatasetManager { public eu.eudat.data.entities.Dataset createOrUpdate(DatasetWizardModel datasetWizardModel, Principal principal) throws Exception { Boolean sendNotification = false; + Dataset tempDataset = null; DMP dmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(datasetWizardModel.getDmp().getId()); if (datasetWizardModel.getId() != null) { - Dataset tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(datasetWizardModel.getId()); + tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(datasetWizardModel.getId()); if (tempDataset != null) { if (datasetWizardModel.getModified().getTime() != tempDataset.getModified().getTime()) { throw new Exception("Dataset has been modified already by another user."); @@ -502,7 +555,7 @@ public class DatasetManager { throw new Exception("DMP is finalized, therefore Dataset cannot be edited."); eu.eudat.data.entities.Dataset dataset = datasetWizardModel.toDataModel(); dataset.setDmp(dmp); - propertiesModelToString(datasetWizardModel, dataset); + dataset.setProperties(propertiesModelToString(datasetWizardModel.getDatasetProfileDefinition())); if (datasetWizardModel.getStatus() == (int) Dataset.Status.FINALISED.getValue()) checkDatasetValidation(dataset); UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); @@ -512,12 +565,13 @@ public class DatasetManager { createExternalDatasetsIfTheyDontExist(dataset); createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), dataset); createServicesIfTheyDontExist(dataset); - Dataset dataset1 = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(dataset); datasetWizardModel.setId(dataset1.getId()); if (datasetWizardModel.getDmp().getGrant() == null) { datasetWizardModel.setDmp(new DataManagementPlan().fromDataModelNoDatasets(dataset1.getDmp())); } + dataset1.setProfile(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(datasetWizardModel.getProfile())); + datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset1)); updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel); if (sendNotification) { if (dataset1.getStatus() != Dataset.Status.FINALISED.getValue()) { @@ -578,27 +632,28 @@ public class DatasetManager { } } - private void propertiesModelToString(DatasetWizardModel datasetWizardModel, eu.eudat.data.entities.Dataset dataset) { + private String propertiesModelToString(PagedDatasetProfile pagedDatasetProfile) { Map values = new HashMap(); - PagedDatasetProfile properties = datasetWizardModel.getDatasetProfileDefinition(); - properties.toMap(values); + pagedDatasetProfile.toMap(values); JSONObject jobject = new JSONObject(values); - dataset.setProperties(jobject.toString()); + return jobject.toString(); } - public void updateTags(DatasetRepository datasetRepository, DatasetWizardModel datasetWizardModel) throws IOException { + public void updateTags(DatasetRepository datasetRepository, DatasetWizardModel datasetWizardModel) throws Exception { // if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) { eu.eudat.elastic.entities.Dataset dataset = new eu.eudat.elastic.entities.Dataset(); dataset.setId(datasetWizardModel.getId().toString()); - DatasetCriteria criteria = new DatasetCriteria(); - criteria.setTags(datasetWizardModel.getTags()); - List tags = apiContext.getOperationsContext().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream) - .filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> datasetWizardModel.getTags().stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList()); - if (tags.isEmpty()) { - datasetWizardModel.getTags().forEach(tag -> tag.setId(UUID.randomUUID().toString())); - dataset.setTags(datasetWizardModel.getTags()); - } else { - dataset.setTags(tags); + if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) { + DatasetCriteria criteria = new DatasetCriteria(); + criteria.setTags(datasetWizardModel.getTags()); + List tags = apiContext.getOperationsContext().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream) + .filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> datasetWizardModel.getTags().stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList()); + if (tags.isEmpty()) { + datasetWizardModel.getTags().forEach(tag -> tag.setId(UUID.randomUUID().toString())); + dataset.setTags(datasetWizardModel.getTags()); + } else { + dataset.setTags(tags); + } } dataset.setLabel(datasetWizardModel.getLabel()); dataset.setDescription(datasetWizardModel.getDescription()); @@ -638,10 +693,13 @@ public class DatasetManager { } dataset.setPublic(datasetWizardModel.getDmp().getPublic()); dataset.setGrantStatus(datasetWizardModel.getDmp().getGrant().getStatus()); + dataset.setFormData(this.getWordDocumentText(datasetWizardModel)); datasetRepository.createOrUpdate(dataset); // } } + + private void createRegistriesIfTheyDontExist(RegistryDao registryDao, eu.eudat.data.entities.Dataset dataset) { if (dataset.getRegistries() != null && !dataset.getRegistries().isEmpty()) { for (eu.eudat.data.entities.Registry registry : dataset.getRegistries()) { @@ -887,11 +945,19 @@ public class DatasetManager { if (principal.getAuthorities().contains(Authorities.ADMIN.getValue())) { this.apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList(); List datasetWizardModels = this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().asQueryable().toList() - .stream().map(dataset -> new DatasetWizardModel().fromDataModel(dataset)).collect(Collectors.toList()); + .stream().map(dataset -> { + DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset); + datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset)); + return datasetWizardModel; + }).collect(Collectors.toList()); datasetWizardModels.forEach(datasetWizardModel -> { try { + eu.eudat.elastic.entities.Dataset dataset = apiContext.getOperationsContext().getDatasetRepository().findDocument(datasetWizardModel.getId().toString()); + if (dataset != null) { + datasetWizardModel.setTags(dataset.getTags()); + } updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel); - } catch (IOException e) { + } catch (Exception e) { logger.error(e.getMessage(), e); } });