package eu.eudat.elastic.repository; import eu.eudat.elastic.criteria.DmpCriteria; import eu.eudat.elastic.entities.Dmp; import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest; import org.elasticsearch.action.get.GetRequest; import org.elasticsearch.action.get.GetResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RestHighLevelClient; import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.client.indices.CreateIndexRequest; import org.elasticsearch.client.indices.GetIndexRequest; import org.elasticsearch.client.indices.PutMappingRequest; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.reindex.DeleteByQueryRequest; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.sort.SortBuilder; import org.elasticsearch.search.sort.SortBuilders; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; import java.io.IOException; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @Service("dmpRepository") public class DmpRepository extends ElasticRepository { private static final Logger logger = LoggerFactory.getLogger(DmpRepository.class); private final Environment environment; @Autowired public DmpRepository(RestHighLevelClient client, Environment environment) { super(client); this.environment = environment; } private void generateMapping() throws IOException { if (this.getClient() != null) { XContentBuilder builder = XContentFactory.jsonBuilder(); builder.startObject(); builder.startObject("properties"); builder.startObject("datasets"); builder.field("type", "nested"); builder.endObject(); builder.endObject(); builder.endObject(); PutMappingRequest putMappingRequest = new PutMappingRequest(this.environment.getProperty("elasticsearch.index")); putMappingRequest.source(builder); this.getClient().indices().putMapping(putMappingRequest, RequestOptions.DEFAULT); } } @Override public Dmp createOrUpdate(Dmp entity) throws IOException { if (this.getClient() != null) { XContentBuilder builder = XContentFactory.jsonBuilder(); IndexRequest request = new IndexRequest(this.environment.getProperty("elasticsearch.index")).id(entity.getId().toString()).source(entity.toElasticEntity(builder)); IndexResponse response = this.getClient().index(request, RequestOptions.DEFAULT); return entity; } return null; } @Override public Dmp findDocument(String id) throws IOException { if (this.getClient() != null) { GetRequest request = new GetRequest(this.environment.getProperty("elasticsearch.index"), id); GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT); return new Dmp().fromElasticEntity(response.getSourceAsMap()); } return null; } @Override public List query(DmpCriteria criteria) throws IOException { if (this.getClient() != null) { SearchRequest searchRequest = new SearchRequest(this.environment.getProperty("elasticsearch.index")); SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index")); countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue())))); CountResponse countResponse = getClient().count(countRequest, RequestOptions.DEFAULT); Long count = countResponse.getCount(); searchSourceBuilder.size(count.intValue()); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue()))); List sortBuilders = new ArrayList<>(); if (criteria.isPublic()) { boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true)); boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue())); } if (criteria.getLike() != null && !criteria.getLike().isEmpty()) { boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{ {Dmp.MapKey.LABEL.getName(), 1.0f}, {Dmp.MapKey.DESCRIPTION.getName(), 1.0f} }).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1])))); } if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getStatus() != null) { boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue())); } if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getGrants() != null && criteria.getGrants().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getRoles() != null && criteria.getRoles().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".role.keyword", criteria.getRoles())); } if (!criteria.isAllowAllVersions()) { boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true)); } if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getGrantStatus() != null) { boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus())); } if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) { boolQuery = boolQuery.should(QueryBuilders.matchAllQuery()); } else { boolQuery.minimumShouldMatch(boolQuery.should().size()); } if (criteria.getSortCriteria() != null && !criteria.getSortCriteria().isEmpty()) { criteria.getSortCriteria().forEach(sortCriteria -> { switch(sortCriteria.getColumnType()) { case COLUMN: sortBuilders.add(SortBuilders.fieldSort(sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name()))); break; case JOIN_COLUMN: List fields = Arrays.asList(sortCriteria.getFieldName().split(":")); fields.forEach(field -> { sortBuilders.add(SortBuilders.fieldSort(sortCriteria.getFieldName()).order(SortOrder.fromString(sortCriteria.getOrderByType().name()))); }); break; } }); } searchSourceBuilder.query(boolQuery).from(criteria.getOffset()).size(criteria.getSize()); sortBuilders.forEach(searchSourceBuilder::sort); searchRequest.source(searchSourceBuilder); SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT); return Arrays.stream(response.getHits().getHits()).map(x -> new Dmp().fromElasticEntity((Map) this.transformFromString(x.getSourceAsString(), Map.class))).collect(Collectors.toList()); } return null; } public Long count(DmpCriteria criteria) throws IOException { if (this.getClient() != null) { CountRequest countRequest = new CountRequest(this.environment.getProperty("elasticsearch.index")); BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery(Dmp.MapKey.STATUS.getName(), Collections.singletonList(Dmp.DMPStatus.DELETED.getValue()))); if (criteria.isPublic()) { boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.ISPUBLIC.getName(), true)); boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), Dmp.DMPStatus.FINALISED.getValue())); } if (criteria.getLike() != null && !criteria.getLike().isEmpty()) { boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][]{ {Dmp.MapKey.LABEL.getName(), 1.0f}, {Dmp.MapKey.DESCRIPTION.getName(), 1.0f} }).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1])))); } if (criteria.getTemplates() != null && criteria.getTemplates().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.TEMPLATES.getName() + ".id.keyword", criteria.getTemplates().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getStatus() != null) { boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.STATUS.getName(), criteria.getStatus().intValue())); } if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GROUPID.getName(), criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getGrants() != null && criteria.getGrants().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.GRANT.getName() + ".keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.COLLABORATORS.getName() + ".id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList()))); } if (!criteria.isAllowAllVersions()) { boolQuery = boolQuery.should(QueryBuilders.termQuery(criteria.isPublic() ? Dmp.MapKey.LASTPUBLICVERSION.getName() : Dmp.MapKey.LASTVERSION.getName(), true)); } if (criteria.getOrganizations() != null && criteria.getOrganizations().size() > 0) { boolQuery = boolQuery.should(QueryBuilders.termsQuery(Dmp.MapKey.ORGANIZATIONS.getName() + ".id.keyword", criteria.getOrganizations().stream().map(UUID::toString).collect(Collectors.toList()))); } if (criteria.getGrantStatus() != null) { boolQuery = boolQuery.should(QueryBuilders.termQuery(Dmp.MapKey.GRANTSTATUS.getName(), criteria.getGrantStatus())); } if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) { boolQuery = boolQuery.should(QueryBuilders.matchAllQuery()); } else { boolQuery.minimumShouldMatch(boolQuery.should().size()); } countRequest.query(boolQuery); CountResponse response = this.getClient().count(countRequest, RequestOptions.DEFAULT); return response.getCount(); } return null; } public boolean createIndex() { try { if (!this.exists()) { CreateIndexRequest createIndexRequest = new CreateIndexRequest(this.environment.getProperty("elasticsearch.index")); this.getClient().indices().create(createIndexRequest, RequestOptions.DEFAULT); this.generateMapping(); } return true; } catch (Exception e) { logger.error(e.getMessage(), e); return false; } } @Override public boolean exists() throws IOException { if (this.getClient() != null) { GetIndexRequest request = new GetIndexRequest(this.environment.getProperty("elasticsearch.index")); return this.getClient().indices().exists(request, RequestOptions.DEFAULT); } return false; } @Override public void clear() throws IOException { if (exists()) { DeleteByQueryRequest delete = new DeleteByQueryRequest(this.environment.getProperty("elasticsearch.index")); delete.setQuery(QueryBuilders.matchAllQuery()); this.getClient().deleteByQuery(delete, RequestOptions.DEFAULT); DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(this.environment.getProperty("elasticsearch.index")); this.getClient().indices().delete(deleteIndexRequest, RequestOptions.DEFAULT); } } }