MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Improvements and bugixes over elastic

This commit is contained in:
George Kalampokis 2020-04-06 18:34:25 +03:00
parent 4d59821b6c
commit d664d19207
6 changed files with 227 additions and 171 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java
View File

@ -39,21 +39,28 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
@Override @Override
public Dataset createOrUpdate(Dataset entity) throws IOException { public Dataset createOrUpdate(Dataset entity) throws IOException {
if (this.getClient() != null) {
XContentBuilder builder = XContentFactory.jsonBuilder(); XContentBuilder builder = XContentFactory.jsonBuilder();
IndexRequest request = new IndexRequest("datasets").id(entity.getId()).source(entity.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder)); IndexRequest request = new IndexRequest("datasets").id(entity.getId()).source(entity.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
this.getClient().index(request, RequestOptions.DEFAULT); this.getClient().index(request, RequestOptions.DEFAULT);
return entity; return entity;
} }
return null;
}
@Override @Override
public Dataset findDocument(String id) throws IOException { public Dataset findDocument(String id) throws IOException {
if (this.getClient() != null) {
GetRequest request = new GetRequest("datasets", id); GetRequest request = new GetRequest("datasets", id);
GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT); GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
return new Dataset().fromElasticEntity(response.getSourceAsMap()); return new Dataset().fromElasticEntity(response.getSourceAsMap());
} }
return null;
}
@Override @Override
public List<Dataset> query(DatasetCriteria criteria) throws IOException { public List<Dataset> query(DatasetCriteria criteria) throws IOException {
if (this.getClient() != null) {
SearchRequest searchRequest = new SearchRequest("datasets"); SearchRequest searchRequest = new SearchRequest("datasets");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@ -130,13 +137,18 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT); SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return Arrays.stream(response.getHits().getHits()).map(x -> this.transformFromString(x.getSourceAsString(), Dataset.class)).collect(Collectors.toList()); return Arrays.stream(response.getHits().getHits()).map(x -> this.transformFromString(x.getSourceAsString(), Dataset.class)).collect(Collectors.toList());
} }
return null;
}
@Override @Override
public boolean exists() throws IOException { public boolean exists() throws IOException {
if (this.getClient() != null) {
GetIndexRequest request = new GetIndexRequest("datasets"); GetIndexRequest request = new GetIndexRequest("datasets");
// request.indices("datasets"); // request.indices("datasets");
return this.getClient().indices().exists(request, RequestOptions.DEFAULT); return this.getClient().indices().exists(request, RequestOptions.DEFAULT);
} }
return false;
}
@Override @Override
public void clear() throws IOException { public void clear() throws IOException {

12
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DmpRepository.java
View File

@ -40,21 +40,28 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
@Override @Override
public Dmp createOrUpdate(Dmp entity) throws IOException { public Dmp createOrUpdate(Dmp entity) throws IOException {
if (this.getClient() != null) {
XContentBuilder builder = XContentFactory.jsonBuilder(); XContentBuilder builder = XContentFactory.jsonBuilder();
IndexRequest request = new IndexRequest("dmps").id(entity.getId().toString()).source(entity.toElasticEntity(builder)); IndexRequest request = new IndexRequest("dmps").id(entity.getId().toString()).source(entity.toElasticEntity(builder));
IndexResponse response = this.getClient().index(request, RequestOptions.DEFAULT); IndexResponse response = this.getClient().index(request, RequestOptions.DEFAULT);
return entity; return entity;
} }
return null;
}
@Override @Override
public Dmp findDocument(String id) throws IOException { public Dmp findDocument(String id) throws IOException {
if (this.getClient() != null) {
GetRequest request = new GetRequest("dmps", id); GetRequest request = new GetRequest("dmps", id);
GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT); GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
return new Dmp().fromElasticEntity(response.getSourceAsMap()); return new Dmp().fromElasticEntity(response.getSourceAsMap());
} }
return null;
}
@Override @Override
public List<Dmp> query(DmpCriteria criteria) throws IOException { public List<Dmp> query(DmpCriteria criteria) throws IOException {
if (this.getClient() != null) {
SearchRequest searchRequest = new SearchRequest("dmps"); SearchRequest searchRequest = new SearchRequest("dmps");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
@ -119,12 +126,17 @@ public class DmpRepository extends ElasticRepository<Dmp, DmpCriteria> {
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT); SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return Arrays.stream(response.getHits().getHits()).map(x -> new Dmp().fromElasticEntity((Map<String, Object>) this.transformFromString(x.getSourceAsString(), Map.class))).collect(Collectors.toList()); return Arrays.stream(response.getHits().getHits()).map(x -> new Dmp().fromElasticEntity((Map<String, Object>) this.transformFromString(x.getSourceAsString(), Map.class))).collect(Collectors.toList());
} }
return null;
}
@Override @Override
public boolean exists() throws IOException { public boolean exists() throws IOException {
if (this.getClient() != null) {
GetIndexRequest request = new GetIndexRequest("dmps"); GetIndexRequest request = new GetIndexRequest("dmps");
return this.getClient().indices().exists(request, RequestOptions.DEFAULT); return this.getClient().indices().exists(request, RequestOptions.DEFAULT);
} }
return false;
}
@Override @Override
public void clear() throws IOException { public void clear() throws IOException {

8
dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/ElasticRepository.java
View File

@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import eu.eudat.elastic.criteria.Criteria; import eu.eudat.elastic.criteria.Criteria;
import eu.eudat.elastic.entities.ElasticEntity; import eu.eudat.elastic.entities.ElasticEntity;
import org.elasticsearch.client.Client; import org.elasticsearch.client.Client;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger; import org.slf4j.Logger;
@ -23,8 +24,15 @@ public abstract class ElasticRepository<T extends ElasticEntity,C extends Criter
} }
public ElasticRepository(RestHighLevelClient client) { public ElasticRepository(RestHighLevelClient client) {
try {
if (client.ping(RequestOptions.DEFAULT)) {
this.client = client; this.client = client;
} }
} catch (IOException e) {
logger.warn("Unable to connect to Elastic Services");
this.client = null;
}
}
public <T> T transformFromString(String value, Class<T> tClass) { public <T> T transformFromString(String value, Class<T> tClass) {
ObjectMapper mapper = new ObjectMapper(); ObjectMapper mapper = new ObjectMapper();

35
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
View File

@ -720,13 +720,17 @@ public class DataManagementPlanManager {
assignFunderUserIfInternal(newDmp, user); assignFunderUserIfInternal(newDmp, user);
assignProjectUserIfInternal(newDmp, user); assignProjectUserIfInternal(newDmp, user);
databaseRepository.getGrantDao().createOrUpdate(newDmp.getGrant()); databaseRepository.getGrantDao().createOrUpdate(newDmp.getGrant());
newDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp); DMP tempDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp);
newDmp.setId(tempDmp.getId());
// Assign creator. // Assign creator.
assignUser(newDmp, user); assignUser(newDmp, user);
copyDatasets(newDmp, databaseRepository.getDatasetDao()); copyDatasets(newDmp, databaseRepository.getDatasetDao());
newDmp.getUsers().clear();
newDmp.setUsers(new HashSet<>(databaseRepository.getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), newDmp.getId())).toList()));
this.updateIndex(newDmp); this.updateIndex(newDmp);
} else { } else {
throw new DMPNewVersionException("Version to update not the latest."); throw new DMPNewVersionException("Version to update not the latest.");
@ -780,7 +784,11 @@ public class DataManagementPlanManager {
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria1).toList().forEach(dmp -> { apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria1).toList().forEach(dmp -> {
for (Dataset dataset: dmp.getDataset()) { for (Dataset dataset: dmp.getDataset()) {
try { try {
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags(); List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (elastic != null) {
tags = elastic.getTags();
}
this.datasetManager.updateTags(dataset, tags); this.datasetManager.updateTags(dataset, tags);
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
@ -930,7 +938,11 @@ public class DataManagementPlanManager {
.thenApplyAsync(entityDataset -> { .thenApplyAsync(entityDataset -> {
Dataset newDataset = new Dataset(); Dataset newDataset = new Dataset();
try { try {
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags(); List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (elastic != null) {
tags = elastic.getTags();
}
this.datasetManager.updateTags(dataset, tags); this.datasetManager.updateTags(dataset, tags);
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
@ -1017,7 +1029,10 @@ public class DataManagementPlanManager {
datasetElastic.setGrantStatus(dataset1.getDmp().getGrant().getStatus()); datasetElastic.setGrantStatus(dataset1.getDmp().getGrant().getStatus());
try { try {
datasetElastic.setTags(apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags()); eu.eudat.elastic.entities.Dataset oldDatasetElastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (oldDatasetElastic != null) {
datasetElastic.setTags(oldDatasetElastic.getTags());
}
datasetElastic.setFormData(this.datasetManager.getWordDocumentText(dataset1)); datasetElastic.setFormData(this.datasetManager.getWordDocumentText(dataset1));
apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(datasetElastic); apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(datasetElastic);
} catch (Exception e) { } catch (Exception e) {
@ -1413,7 +1428,11 @@ public class DataManagementPlanManager {
} }
dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmp.getId())).toList())); dmp.setUsers(new HashSet<>(apiContext.getOperationsContext().getDatabaseRepository().getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmp.getId())).toList()));
try { try {
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags(); List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (elastic != null) {
tags = elastic.getTags();
}
datasetManager.updateTags(dataset, tags); datasetManager.updateTags(dataset, tags);
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);
@ -1455,7 +1474,11 @@ public class DataManagementPlanManager {
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).toList().stream().forEach(dmp1 -> { apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).toList().stream().forEach(dmp1 -> {
dmp1.getDataset().forEach(dataset -> { dmp1.getDataset().forEach(dataset -> {
try { try {
List<Tag> tags = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString()).getTags(); List<Tag> tags = new ArrayList<>();
eu.eudat.elastic.entities.Dataset elastic = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(dataset.getId().toString());
if (elastic != null) {
tags = elastic.getTags();
}
this.datasetManager.updateTags(dataset, tags); this.datasetManager.updateTags(dataset, tags);
} catch (Exception e) { } catch (Exception e) {
logger.error(e.getMessage(), e); logger.error(e.getMessage(), e);

4
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetWizardManager.java
View File

@ -52,9 +52,9 @@ public class DatasetWizardManager {
Dataset oldDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(uuid); Dataset oldDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(uuid);
eu.eudat.elastic.entities.Dataset oldDatasetElasitc = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(uuid.toString()); eu.eudat.elastic.entities.Dataset oldDatasetElasitc = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().findDocument(uuid.toString());
oldDataset.setStatus(Dataset.Status.DELETED.getValue()); oldDataset.setStatus(Dataset.Status.DELETED.getValue());
oldDatasetElasitc.setStatus(oldDataset.getStatus());
apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(oldDataset); apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(oldDataset);
if (uuid != null && oldDatasetElasitc.getId()!= null) { if (oldDatasetElasitc != null && uuid != null && oldDatasetElasitc.getId()!= null) {
oldDatasetElasitc.setStatus(oldDataset.getStatus());
apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(oldDatasetElasitc); apiContext.getOperationsContext().getElasticRepository().getDatasetRepository().createOrUpdate(oldDatasetElasitc);
} }
} }

37
dmp-backend/web/src/main/java/eu/eudat/logic/mapper/elastic/DatasetMapper.java
View File

@ -26,7 +26,8 @@ public class DatasetMapper {
public Dataset toElastic(eu.eudat.data.entities.Dataset dataset, List<Tag> tags) throws Exception { public Dataset toElastic(eu.eudat.data.entities.Dataset dataset, List<Tag> tags) throws Exception {
Dataset elastic = new Dataset(); Dataset elastic = new Dataset();
elastic.setId(dataset.getId().toString()); eu.eudat.data.entities.Dataset tempDataset = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().find(dataset.getId());
elastic.setId(tempDataset.getId().toString());
if (tags != null && !tags.isEmpty()) { if (tags != null && !tags.isEmpty()) {
DatasetCriteria criteria = new DatasetCriteria(); DatasetCriteria criteria = new DatasetCriteria();
criteria.setTags(tags); criteria.setTags(tags);
@ -39,35 +40,35 @@ public class DatasetMapper {
elastic.setTags(tags1); elastic.setTags(tags1);
} }
} }
elastic.setLabel(dataset.getLabel()); elastic.setLabel(tempDataset.getLabel());
elastic.setDescription(dataset.getDescription()); elastic.setDescription(tempDataset.getDescription());
elastic.setTemplate(dataset.getProfile().getId()); elastic.setTemplate(tempDataset.getProfile().getId());
elastic.setStatus(dataset.getStatus()); elastic.setStatus(tempDataset.getStatus());
elastic.setDmp(dataset.getDmp().getId()); elastic.setDmp(tempDataset.getDmp().getId());
elastic.setGroup(dataset.getDmp().getGroupId()); elastic.setGroup(tempDataset.getDmp().getGroupId());
elastic.setGrant(dataset.getDmp().getGrant().getId()); elastic.setGrant(tempDataset.getDmp().getGrant().getId());
if (dataset.getDmp().getUsers() != null) { if (tempDataset.getDmp().getUsers() != null) {
elastic.setCollaborators(dataset.getDmp().getUsers().stream().map(user -> CollaboratorMapper.toElastic(user.getUser())).collect(Collectors.toList())); elastic.setCollaborators(tempDataset.getDmp().getUsers().stream().map(user -> CollaboratorMapper.toElastic(user.getUser())).collect(Collectors.toList()));
} }
DataManagementPlanCriteria dmpCriteria = new DataManagementPlanCriteria(); DataManagementPlanCriteria dmpCriteria = new DataManagementPlanCriteria();
dmpCriteria.setAllVersions(true); dmpCriteria.setAllVersions(true);
dmpCriteria.setGroupIds(Collections.singletonList(dataset.getDmp().getGroupId())); dmpCriteria.setGroupIds(Collections.singletonList(tempDataset.getDmp().getGroupId()));
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream() apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream()
.max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> elastic.setLastVersion(dmp.getId().equals(dataset.getDmp().getId()))); .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> elastic.setLastVersion(dmp.getId().equals(tempDataset.getDmp().getId())));
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream().filter(DMP::isPublic) apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dmpCriteria).toList().stream().filter(DMP::isPublic)
.max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> elastic.setLastPublicVersion(dmp.getId().equals(dataset.getDmp().getId()))); .max(Comparator.comparing(DMP::getVersion)).ifPresent(dmp -> elastic.setLastPublicVersion(dmp.getId().equals(tempDataset.getDmp().getId())));
if (elastic.getLastVersion() == null) { if (elastic.getLastVersion() == null) {
elastic.setLastVersion(true); elastic.setLastVersion(true);
} }
if (elastic.getLastPublicVersion() == null) { if (elastic.getLastPublicVersion() == null) {
elastic.setLastPublicVersion(false); elastic.setLastPublicVersion(false);
} }
if (dataset.getDmp().getOrganisations() != null) { if (tempDataset.getDmp().getOrganisations() != null) {
elastic.setOrganizations(dataset.getDmp().getOrganisations().stream().map(OrganizationMapper::toElastic).collect(Collectors.toList())); elastic.setOrganizations(tempDataset.getDmp().getOrganisations().stream().map(OrganizationMapper::toElastic).collect(Collectors.toList()));
} }
elastic.setPublic(dataset.getDmp().isPublic()); elastic.setPublic(tempDataset.getDmp().isPublic());
elastic.setGrantStatus(dataset.getDmp().getGrant().getStatus()); elastic.setGrantStatus(tempDataset.getDmp().getGrant().getStatus());
elastic.setFormData(datasetManager.getWordDocumentText(dataset)); elastic.setFormData(datasetManager.getWordDocumentText(tempDataset));
return elastic; return elastic;
} }