MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Improvements and bugixes over elastic

This commit is contained in:
George Kalampokis 2020-04-06 18:34:25 +03:00
parent 4d59821b6c
commit d664d19207
6 changed files with 227 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java
View File

@ -39,21 +39,28 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
@Override
public Dataset createOrUpdate(Dataset entity) throws IOException {
if (this.getClient() != null) {
XContentBuilder builder = XContentFactory.jsonBuilder();
IndexRequest request = new IndexRequest("datasets").id(entity.getId()).source(entity.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
this.getClient().index(request, RequestOptions.DEFAULT);
return entity;
}
return null;
}
@Override
public Dataset findDocument(String id) throws IOException {
GetRequest request = new GetRequest("datasets",id);
if (this.getClient() != null) {
GetRequest request = new GetRequest("datasets", id);
GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
return new Dataset().fromElasticEntity(response.getSourceAsMap());
}
return null;
}
@Override
public List<Dataset> query(DatasetCriteria criteria) throws IOException {
if (this.getClient() != null) {
SearchRequest searchRequest = new SearchRequest("datasets");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@ -71,10 +78,10 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
boolQuery = boolQuery.should(QueryBuilders.termQuery("lastPublicVersion.keyword", "true"));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][] {
{ "label", 1.0f },
{ "description", 1.0f },
{ "formData", 1.0f }
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
{"label", 1.0f},
{"description", 1.0f},
{"formData", 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
@ -130,13 +137,18 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return Arrays.stream(response.getHits().getHits()).map(x -> this.transformFromString(x.getSourceAsString(), Dataset.class)).collect(Collectors.toList());
}
return null;
}
@Override
public boolean exists() throws IOException {
if (this.getClient() != null) {
GetIndexRequest request = new GetIndexRequest("datasets");
// request.indices("datasets");
return this.getClient().indices().exists(request, RequestOptions.DEFAULT);
}
return false;
}
@Override
public void clear() throws IOException {

22
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DmpRepository.java
View File

@ -40,21 +40,28 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
@Override
public Dmp createOrUpdate(Dmp entity) throws IOException {
if (this.getClient() != null) {
XContentBuilder builder = XContentFactory.jsonBuilder();
IndexRequest request = new IndexRequest("dmps").id(entity.getId().toString()).source(entity.toElasticEntity(builder));
IndexResponse response = this.getClient().index(request, RequestOptions.DEFAULT);
return entity;
}
return null;
}
@Override
public Dmp findDocument(String id) throws IOException {
GetRequest request = new GetRequest("dmps",id);
if (this.getClient() != null) {
GetRequest request = new GetRequest("dmps", id);
GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
return new Dmp().fromElasticEntity(response.getSourceAsMap());
}
return null;
}
@Override
public List<Dmp> query(DmpCriteria criteria) throws IOException {
if (this.getClient() != null) {
SearchRequest searchRequest = new SearchRequest("dmps");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@ -71,9 +78,9 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue()));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][] {
{ Dmp.MapKey.LABEL.getName(), 1.0f },
{ Dmp.MapKey.DESCRIPTION.getName(), 1.0f }
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{
{Dmp.MapKey.LABEL.getName(), 1.0f},
{Dmp.MapKey.DESCRIPTION.getName(), 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
@ -117,14 +124,19 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
searchSourceBuilder.query(boolQuery);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return Arrays.stream(response.getHits().getHits()).map(x -> new Dmp().fromElasticEntity((Map<String, Object>)this.transformFromString(x.getSourceAsString(), Map.class))).collect(Collectors.toList());
return Arrays.stream(response.getHits().getHits()).map(x -> new Dmp().fromElasticEntity((Map<String, Object>) this.transformFromString(x.getSourceAsString(), Map.class))).collect(Collectors.toList());
}
return null;
}
@Override
public boolean exists() throws IOException {
if (this.getClient() != null) {
GetIndexRequest request = new GetIndexRequest("dmps");
return this.getClient().indices().exists(request, RequestOptions.DEFAULT);
}
return false;
}
@Override
public void clear() throws IOException {

8
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/ElasticRepository.java
View File

@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.elastic.criteria.Criteria;
import eu.eudat.elastic.entities.ElasticEntity;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
@ -23,8 +24,15 @@ public abstract class ElasticRepository<T extends ElasticEntity,C extends Criter
}
public ElasticRepository(RestHighLevelClient client) {
try {
if (client.ping(RequestOptions.DEFAULT)) {
this.client = client;
}
} catch (IOException e) {
logger.warn("Unable to connect to Elastic Services");
this.client = null;
}
}
public <T> T transformFromString(String value, Class<T> tClass) {
ObjectMapper mapper = new ObjectMapper();

35
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
View File

@ -720,13 +720,17 @@ public class DataManagementPlanManager {
assignFunderUserIfInternal(newDmp, user);
assignProjectUserIfInternal(newDmp, user);
databaseRepository.getGrantDao().createOrUpdate(newDmp.getGrant());
newDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp);
DMP tempDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp);
newDmp.setId(tempDmp.getId());
// Assign creator.
assignUser(newDmp, user);
copyDatasets(newDmp, databaseRepository.getDatasetDao());
newDmp.getUsers().clear();
newDmp.setUsers(new HashSet<>(databaseRepository.getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), newDmp.getId())).toList()));
this.updateIndex(newDmp);
} else {
throw new DMPNewVersionException("Version to update not the latest.");
@ -780,7 +784,11 @@ public class DataManagementPlanManager {
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria1).toList().forEach(dmp -> {
for (Dataset dataset: dmp.getDataset()) {
try {
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags();
List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (elastic != null) {
tags = elastic.getTags();
}
this.datasetManager.updateTags(dataset, tags);
} catch (Exception e) {
logger.error(e.getMessage(), e);
@ -930,7 +938,11 @@ public class DataManagementPlanManager {
.thenApplyAsync(entityDataset -> {
Dataset newDataset = new Dataset();
try {
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags();
List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (elastic != null) {
tags = elastic.getTags();
}
this.datasetManager.updateTags(dataset, tags);
} catch (Exception e) {
logger.error(e.getMessage(), e);
@ -1017,7 +1029,10 @@ public class DataManagementPlanManager {
datasetElastic.setGrantStatus(dataset1.getDmp().getGrant().getStatus());
try {
datasetElastic.setTags(apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags());
eu.eudat.elastic.entities.Dataset oldDatasetElastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (oldDatasetElastic != null) {
datasetElastic.setTags(oldDatasetElastic.getTags());
}
datasetElastic.setFormData(this.datasetManager.getWordDocumentText(dataset1));
apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(datasetElastic);
} catch (Exception e) {
@ -1413,7 +1428,11 @@ public class DataManagementPlanManager {
}
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmp.getId())).toList()));
try {
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags();
List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (elastic != null) {
tags = elastic.getTags();
}
datasetManager.updateTags(dataset, tags);
} catch (Exception e) {
logger.error(e.getMessage(), e);
@ -1455,7 +1474,11 @@ public class DataManagementPlanManager {
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).toList().stream().forEach(dmp1 -> {
dmp1.getDataset().forEach(dataset -> {
try {
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags();
List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (elastic != null) {
tags = elastic.getTags();
}
this.datasetManager.updateTags(dataset, tags);
} catch (Exception e) {
logger.error(e.getMessage(), e);

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetWizardManager.java
View File

@ -52,9 +52,9 @@ public class DatasetWizardManager {
Dataset oldDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(uuid);
eu.eudat.elastic.entities.Dataset oldDatasetElasitc = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(uuid.toString());
oldDataset.setStatus(Dataset.Status.DELETED.getValue());
oldDatasetElasitc.setStatus(oldDataset.getStatus());
apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(oldDataset);
if (uuid != null && oldDatasetElasitc.getId()!= null) {
if (oldDatasetElasitc != null && uuid != null && oldDatasetElasitc.getId()!= null) {
oldDatasetElasitc.setStatus(oldDataset.getStatus());
apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(oldDatasetElasitc);
}
}

37
dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/DatasetMapper.java
View File

@ -26,7 +26,8 @@ public class DatasetMapper {
public Dataset toElastic(eu.eudat.data.entities.Dataset dataset, List<Tag> tags) throws Exception {
Dataset elastic = new Dataset();
elastic.setId(dataset.getId().toString());
eu.eudat.data.entities.Dataset tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(dataset.getId());
elastic.setId(tempDataset.getId().toString());
if (tags != null && !tags.isEmpty()) {
DatasetCriteria criteria = new DatasetCriteria();
criteria.setTags(tags);
@ -39,35 +40,35 @@ public class DatasetMapper {
elastic.setTags(tags1);
}
}
elastic.setLabel(dataset.getLabel());
elastic.setDescription(dataset.getDescription());
elastic.setTemplate(dataset.getProfile().getId());
elastic.setStatus(dataset.getStatus());
elastic.setDmp(dataset.getDmp().getId());
elastic.setGroup(dataset.getDmp().getGroupId());
elastic.setGrant(dataset.getDmp().getGrant().getId());
if (dataset.getDmp().getUsers() != null) {
elastic.setCollaborators(dataset.getDmp().getUsers().stream().map(user -> CollaboratorMapper.toElastic(user.getUser())).collect(Collectors.toList()));
elastic.setLabel(tempDataset.getLabel());
elastic.setDescription(tempDataset.getDescription());
elastic.setTemplate(tempDataset.getProfile().getId());
elastic.setStatus(tempDataset.getStatus());
elastic.setDmp(tempDataset.getDmp().getId());
elastic.setGroup(tempDataset.getDmp().getGroupId());
elastic.setGrant(tempDataset.getDmp().getGrant().getId());
if (tempDataset.getDmp().getUsers() != null) {
elastic.setCollaborators(tempDataset.getDmp().getUsers().stream().map(user -> CollaboratorMapper.toElastic(user.getUser())).collect(Collectors.toList()));
}
DataManagementPlanCriteria dmpCriteria = new DataManagementPlanCriteria();
dmpCriteria.setAllVersions(true);
dmpCriteria.setGroupIds(Collections.singletonList(dataset.getDmp().getGroupId()));
dmpCriteria.setGroupIds(Collections.singletonList(tempDataset.getDmp().getGroupId()));
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream()
.max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> elastic.setLastVersion(dmp.getId().equals(dataset.getDmp().getId())));
.max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> elastic.setLastVersion(dmp.getId().equals(tempDataset.getDmp().getId())));
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream().filter(DMP::isPublic)
.max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> elastic.setLastPublicVersion(dmp.getId().equals(dataset.getDmp().getId())));
.max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> elastic.setLastPublicVersion(dmp.getId().equals(tempDataset.getDmp().getId())));
if (elastic.getLastVersion() == null) {
elastic.setLastVersion(true);
}
if (elastic.getLastPublicVersion() == null) {
elastic.setLastPublicVersion(false);
}
if (dataset.getDmp().getOrganisations() != null) {
elastic.setOrganizations(dataset.getDmp().getOrganisations().stream().map(OrganizationMapper::toElastic).collect(Collectors.toList()));
if (tempDataset.getDmp().getOrganisations() != null) {
elastic.setOrganizations(tempDataset.getDmp().getOrganisations().stream().map(OrganizationMapper::toElastic).collect(Collectors.toList()));
}
elastic.setPublic(dataset.getDmp().isPublic());
elastic.setGrantStatus(dataset.getDmp().getGrant().getStatus());
elastic.setFormData(datasetManager.getWordDocumentText(dataset));
elastic.setPublic(tempDataset.getDmp().isPublic());
elastic.setGrantStatus(tempDataset.getDmp().getGrant().getStatus());
elastic.setFormData(datasetManager.getWordDocumentText(tempDataset));
return elastic;
}