MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Add Form Data index for dataset

This commit is contained in:
George Kalampokis 2020-03-18 18:23:34 +02:00
parent 7082b958ac
commit aa1156f940
5 changed files with 121 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dataset.java
View File

@ -58,6 +58,7 @@ public class Dataset implements ElasticEntity<Dataset> {
private List<Organization> organizations;
private Boolean isPublic;
private Short grantStatus;
private String formData;
public String getId() {
return id;
@ -179,6 +180,14 @@ public class Dataset implements ElasticEntity<Dataset> {
this.grantStatus = grantStatus;
}
public String getFormData() {
return formData;
}
public void setFormData(String formData) {
this.formData = formData;
}
@Override
public XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException {
builder.startObject();
@ -235,6 +244,7 @@ public class Dataset implements ElasticEntity<Dataset> {
if (this.grantStatus != null) {
builder.field("grantStatus", this.grantStatus.toString());
}
builder.field("formData", this.formData);
builder.endObject();
return builder;
}
@ -267,6 +277,7 @@ public class Dataset implements ElasticEntity<Dataset> {
if (fields.get("grantStatus") != null) {
this.grantStatus = Short.valueOf((String) fields.get("grantStatus"));
}
this.formData = (String) fields.get("formData");
}
return this;
}

3
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java
View File

@ -71,9 +71,10 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
boolQuery = boolQuery.should(QueryBuilders.termQuery("lastPublicVersion.keyword", "true"));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery("*" + criteria.getLike() + "*").fields(Stream.of(new Object[][] {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][] {
{ "label", 1.0f },
{ "description", 1.0f },
{ "formData", 1.0f }
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}

4
dmp-backend/web/src/main/java/eu/eudat/controllers/DatasetWizardController.java
View File

@ -88,7 +88,7 @@ public class DatasetWizardController extends BaseController {
VisibilityRuleService visibilityRuleService = this.getApiContext().getUtilitiesService().getVisibilityRuleService();
return this.datasetManager.getDocument(id, visibilityRuleService, contentType, principal);
} else if (contentType.equals("application/msword")) {
File file = datasetManager.getWordDocument(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal);
File file = datasetManager.getWordDocumentFile(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal);
InputStream resource = new FileInputStream(file);
HttpHeaders responseHeaders = new HttpHeaders();
responseHeaders.setContentLength(file.length());
@ -152,7 +152,7 @@ public class DatasetWizardController extends BaseController {
@RequestMapping(method = RequestMethod.GET, value = {"/getPDF/{id}"})
public @ResponseBody
ResponseEntity<byte[]> getPDFDocument(@PathVariable String id, Principal principal) throws IllegalAccessException, IOException, InstantiationException, InterruptedException {
File file = datasetManager.getWordDocument(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal);
File file = datasetManager.getWordDocumentFile(this.configLoader, id, this.getApiContext().getUtilitiesService().getVisibilityRuleService(), principal);
String fileName = file.getName();
if (fileName.endsWith(".docx")){
fileName = fileName.substring(0, fileName.length() - 5);

15
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
View File

@ -733,8 +733,9 @@ public class DataManagementPlanManager {
try {
DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset);
datasetWizardModel.setTags(apiContext.getOperationsContext().getDatasetRepository().findDocument(dataset.getId().toString()).getTags());
datasetWizardModel.setDatasetProfileDefinition(this.datasetManager.getPagedProfile(datasetWizardModel, dataset));
this.datasetManager.updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel);
} catch (IOException e) {
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
@ -884,8 +885,9 @@ public class DataManagementPlanManager {
try {
DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(entityDataset);
datasetWizardModel.setTags(apiContext.getOperationsContext().getDatasetRepository().findDocument(dataset.getId().toString()).getTags());
datasetWizardModel.setDatasetProfileDefinition(this.datasetManager.getPagedProfile(datasetWizardModel, entityDataset));
this.datasetManager.updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel);
} catch (IOException e) {
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
newDataset.update(entityDataset);
@ -971,8 +973,12 @@ public class DataManagementPlanManager {
try {
datasetElastic.setTags(apiContext.getOperationsContext().getDatasetRepository().findDocument(dataset.getId().toString()).getTags());
DatasetWizardModel datasetWizardModel = new DatasetWizardModel();
datasetWizardModel.setStatus(dataset1.getStatus());
datasetWizardModel.setDatasetProfileDefinition(this.datasetManager.getPagedProfile(datasetWizardModel, dataset1));
datasetElastic.setFormData(this.datasetManager.getWordDocumentText(datasetWizardModel));
apiContext.getOperationsContext().getDatasetRepository().createOrUpdate(datasetElastic);
} catch (IOException e) {
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
})));
@ -1343,8 +1349,9 @@ public class DataManagementPlanManager {
try {
DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset);
datasetWizardModel.setTags(apiContext.getOperationsContext().getDatasetRepository().findDocument(dataset.getId().toString()).getTags());
datasetWizardModel.setDatasetProfileDefinition(this.datasetManager.getPagedProfile(datasetWizardModel, dataset));
this.datasetManager.updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel);
} catch (IOException e) {
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
});

124
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java
View File

@ -43,6 +43,7 @@ import eu.eudat.models.data.user.composite.PagedDatasetProfile;
import eu.eudat.queryable.QueryableList;
import eu.eudat.types.Authorities;
import org.apache.commons.io.IOUtils;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import org.apache.poi.xwpf.usermodel.XWPFDocument;
import org.apache.poi.xwpf.usermodel.XWPFParagraph;
import org.apache.poi.xwpf.usermodel.XWPFRun;
@ -64,6 +65,7 @@ import org.w3c.dom.NodeList;
import javax.activation.MimetypesFileTypeMap;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import javax.transaction.Transactional;
import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBException;
import javax.xml.bind.Unmarshaller;
@ -96,14 +98,16 @@ public class DatasetManager {
private DatasetRepository datasetRepository;
private BuilderFactory builderFactory;
private UserManager userManager;
private ConfigLoader configLoader;
@Autowired
public DatasetManager(ApiContext apiContext, UserManager userManager) {
public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader) {
this.apiContext = apiContext;
this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
this.datasetRepository = apiContext.getOperationsContext().getDatasetRepository();
this.builderFactory = apiContext.getOperationsContext().getBuilderFactory();
this.userManager = userManager;
this.configLoader = configLoader;
}
public DataTableData<DatasetListingModel> getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception {
@ -331,15 +335,11 @@ public class DatasetManager {
return pagedDatasetProfile;
}
public File getWordDocument(ConfigLoader configLoader, String id, VisibilityRuleService visibilityRuleService, Principal principal) throws IOException {
private XWPFDocument getWordDocument(ConfigLoader configLoader, eu.eudat.data.entities.Dataset datasetEntity, VisibilityRuleService visibilityRuleService) throws IOException {
WordBuilder wordBuilder = new WordBuilder();
DatasetWizardModel dataset = new DatasetWizardModel();
XWPFDocument document = configLoader.getDocument();
eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers()
.stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId())
.collect(Collectors.toList()).size() == 0)
throw new UnauthorisedException();
wordBuilder.addParagraphContent(datasetEntity.getLabel(), document, ParagraphStyle.HEADER1, BigInteger.ZERO);
// Space below Dataset title.
@ -410,10 +410,62 @@ public class DatasetManager {
document.removeBodyElement(0);
}
return document;
//FileOutputStream out = new FileOutputStream(exportFile);
// document.write(out);
// out.close();
// return exportFile;
}
private XWPFDocument getightWordDocument(ConfigLoader configLoader, DatasetWizardModel dataset, VisibilityRuleService visibilityRuleService) throws IOException {
WordBuilder wordBuilder = new WordBuilder();
XWPFDocument document = configLoader.getDocument();
// Space below Dataset title.
XWPFParagraph parBreakDataset = document.createParagraph();
Map<String, Object> properties = new HashMap<>();
if (dataset.getDatasetProfileDefinition() != null) {
JSONObject jObject = new JSONObject(propertiesModelToString(dataset.getDatasetProfileDefinition()));
properties = jObject.toMap();
}
wordBuilder.addParagraphContent("Dataset Description", document, ParagraphStyle.HEADER2, BigInteger.ZERO);
visibilityRuleService.setProperties(properties);
visibilityRuleService.buildVisibilityContext(dataset.getDatasetProfileDefinition().getRules());
wordBuilder.build(document, dataset.getDatasetProfileDefinition(), visibilityRuleService);
// Removes the top empty headings.
for (int i = 0; i < 6; i++) {
document.removeBodyElement(0);
}
return document;
//FileOutputStream out = new FileOutputStream(exportFile);
// document.write(out);
// out.close();
// return exportFile;
}
public File getWordDocumentFile(ConfigLoader configLoader, String id, VisibilityRuleService visibilityRuleService, Principal principal) throws IOException {
eu.eudat.data.entities.Dataset datasetEntity = databaseRepository.getDatasetDao().find(UUID.fromString(id), HintedModelFactory.getHint(DatasetWizardModel.class));
if (!datasetEntity.getDmp().isPublic() && datasetEntity.getDmp().getUsers()
.stream().filter(userInfo -> userInfo.getUser().getId() == principal.getId())
.collect(Collectors.toList()).size() == 0)
throw new UnauthorisedException();
String label = datasetEntity.getLabel().replaceAll("[^a-zA-Z0-9+ ]", "");
File exportFile = new File(label + ".docx");
XWPFDocument document = getWordDocument(configLoader, datasetEntity, visibilityRuleService);
FileOutputStream out = new FileOutputStream(exportFile);
document.write(out);
out.close();
return exportFile;
document.write(out);
out.close();
return exportFile;
}
public String getWordDocumentText (DatasetWizardModel datasetEntity) throws Exception {
XWPFDocument document = getightWordDocument(this.configLoader, datasetEntity, this.apiContext.getUtilitiesService().getVisibilityRuleService());
XWPFWordExtractor extractor = new XWPFWordExtractor(document);
return extractor.getText();/*.replaceAll("\n\\s*", " ");*/
}
public FileEnvelope getXmlDocument(String id, VisibilityRuleService visibilityRuleService, Principal principal) throws InstantiationException, IllegalAccessException, IOException {
@ -488,9 +540,10 @@ public class DatasetManager {
public eu.eudat.data.entities.Dataset createOrUpdate(DatasetWizardModel datasetWizardModel, Principal principal) throws Exception {
Boolean sendNotification = false;
Dataset tempDataset = null;
DMP dmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(datasetWizardModel.getDmp().getId());
if (datasetWizardModel.getId() != null) {
Dataset tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(datasetWizardModel.getId());
tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(datasetWizardModel.getId());
if (tempDataset != null) {
if (datasetWizardModel.getModified().getTime() != tempDataset.getModified().getTime()) {
throw new Exception("Dataset has been modified already by another user.");
@ -502,7 +555,7 @@ public class DatasetManager {
throw new Exception("DMP is finalized, therefore Dataset cannot be edited.");
eu.eudat.data.entities.Dataset dataset = datasetWizardModel.toDataModel();
dataset.setDmp(dmp);
propertiesModelToString(datasetWizardModel, dataset);
dataset.setProperties(propertiesModelToString(datasetWizardModel.getDatasetProfileDefinition()));
if (datasetWizardModel.getStatus() == (int) Dataset.Status.FINALISED.getValue())
checkDatasetValidation(dataset);
UserInfo userInfo = apiContext.getOperationsContext().getBuilderFactory().getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
@ -512,12 +565,13 @@ public class DatasetManager {
createExternalDatasetsIfTheyDontExist(dataset);
createRegistriesIfTheyDontExist(apiContext.getOperationsContext().getDatabaseRepository().getRegistryDao(), dataset);
createServicesIfTheyDontExist(dataset);
Dataset dataset1 = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(dataset);
datasetWizardModel.setId(dataset1.getId());
if (datasetWizardModel.getDmp().getGrant() == null) {
datasetWizardModel.setDmp(new DataManagementPlan().fromDataModelNoDatasets(dataset1.getDmp()));
}
dataset1.setProfile(this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(datasetWizardModel.getProfile()));
datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset1));
updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel);
if (sendNotification) {
if (dataset1.getStatus() != Dataset.Status.FINALISED.getValue()) {
@ -578,27 +632,28 @@ public class DatasetManager {
}
}
private void propertiesModelToString(DatasetWizardModel datasetWizardModel, eu.eudat.data.entities.Dataset dataset) {
private String propertiesModelToString(PagedDatasetProfile pagedDatasetProfile) {
Map<String, Object> values = new HashMap();
PagedDatasetProfile properties = datasetWizardModel.getDatasetProfileDefinition();
properties.toMap(values);
pagedDatasetProfile.toMap(values);
JSONObject jobject = new JSONObject(values);
dataset.setProperties(jobject.toString());
return jobject.toString();
}
public void updateTags(DatasetRepository datasetRepository, DatasetWizardModel datasetWizardModel) throws IOException {
public void updateTags(DatasetRepository datasetRepository, DatasetWizardModel datasetWizardModel) throws Exception {
// if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) {
eu.eudat.elastic.entities.Dataset dataset = new eu.eudat.elastic.entities.Dataset();
dataset.setId(datasetWizardModel.getId().toString());
DatasetCriteria criteria = new DatasetCriteria();
criteria.setTags(datasetWizardModel.getTags());
List<Tag> tags = apiContext.getOperationsContext().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
.filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> datasetWizardModel.getTags().stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
if (tags.isEmpty()) {
datasetWizardModel.getTags().forEach(tag -> tag.setId(UUID.randomUUID().toString()));
dataset.setTags(datasetWizardModel.getTags());
} else {
dataset.setTags(tags);
if (datasetWizardModel.getTags() != null && !datasetWizardModel.getTags().isEmpty()) {
DatasetCriteria criteria = new DatasetCriteria();
criteria.setTags(datasetWizardModel.getTags());
List<Tag> tags = apiContext.getOperationsContext().getDatasetRepository().query(criteria).stream().map(eu.eudat.elastic.entities.Dataset::getTags).flatMap(Collection::stream)
.filter(StreamDistinctBy.distinctByKey(Tag::getId)).filter(tag -> datasetWizardModel.getTags().stream().anyMatch(tag1 -> tag1.getName().equals(tag.getName()))).collect(Collectors.toList());
if (tags.isEmpty()) {
datasetWizardModel.getTags().forEach(tag -> tag.setId(UUID.randomUUID().toString()));
dataset.setTags(datasetWizardModel.getTags());
} else {
dataset.setTags(tags);
}
}
dataset.setLabel(datasetWizardModel.getLabel());
dataset.setDescription(datasetWizardModel.getDescription());
@ -638,10 +693,13 @@ public class DatasetManager {
}
dataset.setPublic(datasetWizardModel.getDmp().getPublic());
dataset.setGrantStatus(datasetWizardModel.getDmp().getGrant().getStatus());
dataset.setFormData(this.getWordDocumentText(datasetWizardModel));
datasetRepository.createOrUpdate(dataset);
// }
}
private void createRegistriesIfTheyDontExist(RegistryDao registryDao, eu.eudat.data.entities.Dataset dataset) {
if (dataset.getRegistries() != null && !dataset.getRegistries().isEmpty()) {
for (eu.eudat.data.entities.Registry registry : dataset.getRegistries()) {
@ -887,11 +945,19 @@ public class DatasetManager {
if (principal.getAuthorities().contains(Authorities.ADMIN.getValue())) {
this.apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().asQueryable().toList();
List<DatasetWizardModel> datasetWizardModels = this.apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().asQueryable().toList()
.stream().map(dataset -> new DatasetWizardModel().fromDataModel(dataset)).collect(Collectors.toList());
.stream().map(dataset -> {
DatasetWizardModel datasetWizardModel = new DatasetWizardModel().fromDataModel(dataset);
datasetWizardModel.setDatasetProfileDefinition(getPagedProfile(datasetWizardModel, dataset));
return datasetWizardModel;
}).collect(Collectors.toList());
datasetWizardModels.forEach(datasetWizardModel -> {
try {
eu.eudat.elastic.entities.Dataset dataset = apiContext.getOperationsContext().getDatasetRepository().findDocument(datasetWizardModel.getId().toString());
if (dataset != null) {
datasetWizardModel.setTags(dataset.getTags());
}
updateTags(apiContext.getOperationsContext().getDatasetRepository(), datasetWizardModel);
} catch (IOException e) {
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
});