From 81ed82683d0a831eff10d2ccb467f757e18c7039 Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Tue, 28 Jul 2020 11:30:06 +0200 Subject: [PATCH] optimizations of es settings --- .../eu/dnetlib/lbs/elasticsearch/Event.java | 6 +-- .../lbs/elasticsearch/Notification.java | 20 +++---- .../matchers/SubscriptionEventMatcher.java | 2 - .../properties/ElasticSearchProperties.java | 20 ------- .../src/main/resources/application.properties | 3 -- .../src/main/resources/config/mapping_es7.sh | 54 +++++++++++++++++++ .../main/resources/static/html/summary.html | 8 +-- 7 files changed, 69 insertions(+), 44 deletions(-) diff --git a/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/elasticsearch/Event.java b/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/elasticsearch/Event.java index 75b69940..91e9cf21 100644 --- a/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/elasticsearch/Event.java +++ b/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/elasticsearch/Event.java @@ -7,7 +7,7 @@ import org.springframework.data.elasticsearch.annotations.Document; import org.springframework.data.elasticsearch.annotations.Field; import org.springframework.data.elasticsearch.annotations.FieldType; -@Document(indexName = "#{elasticSearchProperties.eventsIndexName}", type = "#{elasticSearchProperties.eventsIndexType}") +@Document(indexName = "#{elasticSearchProperties.eventsIndexName}", shards = 48, replicas = 0, refreshInterval = "600s") public class Event { @Id @@ -37,8 +37,8 @@ public class Event { public Event() {} public Event(final String producerId, final String eventId, final String topic, final String payload, final Long creationDate, final Long expiryDate, - final boolean instantMessage, - final Map map) { + final boolean instantMessage, + final Map map) { this.producerId = producerId; this.eventId = eventId; this.topic = topic; diff --git a/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/elasticsearch/Notification.java b/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/elasticsearch/Notification.java index 8bd194f1..a49b42a2 100644 --- a/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/elasticsearch/Notification.java +++ b/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/elasticsearch/Notification.java @@ -11,7 +11,7 @@ import org.springframework.data.elasticsearch.annotations.FieldType; import eu.dnetlib.lbs.subscriptions.Subscription; -@Document(indexName = "#{elasticSearchProperties.notificationsIndexName}", type = "#{elasticSearchProperties.notificationsIndexType}") +@Document(indexName = "#{elasticSearchProperties.notificationsIndexName}", shards = 48, replicas = 0, refreshInterval = "600s") public class Notification { @Id @@ -41,8 +41,8 @@ public class Notification { public Notification() {} public Notification(final String notificationId, final String subscriptionId, final String producerId, final String eventId, final String topic, - final String payload, - final Map map, final Long date) { + final String payload, + final Map map, final Long date) { super(); this.notificationId = notificationId; this.subscriptionId = subscriptionId; @@ -56,13 +56,13 @@ public class Notification { public Notification(final Subscription s, final Event e) { this(Notification.calculateId(s, e), - s.getSubscriptionId(), - e.getProducerId(), - e.getEventId(), - e.getTopic(), - e.getPayload(), - e.getMap(), - new Date().getTime()); + s.getSubscriptionId(), + e.getProducerId(), + e.getEventId(), + e.getTopic(), + e.getPayload(), + e.getMap(), + new Date().getTime()); } public String getNotificationId() { diff --git a/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/matchers/SubscriptionEventMatcher.java b/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/matchers/SubscriptionEventMatcher.java index d4cf2555..0bf1fea9 100644 --- a/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/matchers/SubscriptionEventMatcher.java +++ b/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/matchers/SubscriptionEventMatcher.java @@ -110,8 +110,6 @@ public class SubscriptionEventMatcher implements Runnable { .must(QueryBuilders.rangeQuery("creationDate").from(s.getLastNotificationDate() != null ? s.getLastNotificationDate().getTime() : 0)) .must(QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None))) .withSearchType(SearchType.DEFAULT) - // .withIndices(elasticSearchProperties.getEventsIndexName()) - // .withTypes(elasticSearchProperties.getEventsIndexType()) .withPageable(PageRequest.of(0, 10)) .build(); diff --git a/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/properties/ElasticSearchProperties.java b/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/properties/ElasticSearchProperties.java index 9028ea76..900bb9af 100644 --- a/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/properties/ElasticSearchProperties.java +++ b/apps/dhp-broker-application/src/main/java/eu/dnetlib/lbs/properties/ElasticSearchProperties.java @@ -16,12 +16,8 @@ public class ElasticSearchProperties { @NotNull private String eventsIndexName; @NotNull - private String eventsIndexType; - @NotNull private String notificationsIndexName; @NotNull - private String notificationsIndexType; - @NotNull private long connectionTimeout; @NotNull private long socketTimeout; @@ -50,14 +46,6 @@ public class ElasticSearchProperties { this.eventsIndexName = eventsIndexName; } - public String getEventsIndexType() { - return this.eventsIndexType; - } - - public void setEventsIndexType(final String eventsIndexType) { - this.eventsIndexType = eventsIndexType; - } - public String getNotificationsIndexName() { return this.notificationsIndexName; } @@ -66,14 +54,6 @@ public class ElasticSearchProperties { this.notificationsIndexName = notificationsIndexName; } - public String getNotificationsIndexType() { - return this.notificationsIndexType; - } - - public void setNotificationsIndexType(final String notificationsIndexType) { - this.notificationsIndexType = notificationsIndexType; - } - public long getConnectionTimeout() { return connectionTimeout; } diff --git a/apps/dhp-broker-application/src/main/resources/application.properties b/apps/dhp-broker-application/src/main/resources/application.properties index f9eaa82a..1d1b573e 100644 --- a/apps/dhp-broker-application/src/main/resources/application.properties +++ b/apps/dhp-broker-application/src/main/resources/application.properties @@ -27,11 +27,8 @@ lbs.elastic.clusterNodes = 127.0.0.1:9200 lbs.elastic.homepage = http:// lbs.elastic.socketTimeout = 30000 lbs.elastic.connectionTimeout = 30000 - lbs.elastic.eventsIndexName = oa_events_beta -lbs.elastic.eventsIndexType = event lbs.elastic.notificationsIndexName = oa_notifications_beta -lbs.elastic.notificationsIndexType = notification lbs.mail.from = notifications-noreply@openaire.eu lbs.mail.fromName = Literature Broker Service diff --git a/apps/dhp-broker-application/src/main/resources/config/mapping_es7.sh b/apps/dhp-broker-application/src/main/resources/config/mapping_es7.sh index f40cd1e5..6fec7b32 100755 --- a/apps/dhp-broker-application/src/main/resources/config/mapping_es7.sh +++ b/apps/dhp-broker-application/src/main/resources/config/mapping_es7.sh @@ -4,6 +4,57 @@ INDEXBASEURL=http://localhost:9200 EVENTINDEXNAME=oa_events_beta NOTIFICATIONINDEXNAME=oa_notifications_beta + +echo +echo "Update settings of events index..." +curl -XPOST "$INDEXBASEURL/$EVENTINDEXNAME/_close" +echo +curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$EVENTINDEXNAME/_settings" -d ' +{ + "index": { + "translog": { + "sync_interval": "15s", + "durability": "ASYNC" + }, + "analysis": { + "analyzer": { + "analyzer_keyword": { + "filter": "lowercase", + "tokenizer": "keyword" + } + } + } + } +}' +echo +curl -XPOST "$INDEXBASEURL/$EVENTINDEXNAME/_open" + +echo +echo +echo "Update settings of notifications index..." +curl -XPOST "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_close" +echo +curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_settings" -d ' +{ + "index": { + "translog": { + "sync_interval": "15s", + "durability": "ASYNC" + }, + "analysis": { + "analyzer": { + "analyzer_keyword": { + "filter": "lowercase", + "tokenizer": "keyword" + } + } + } + } +}' +echo +curl -XPOST "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_open" + +echo echo echo "Fixing events index..." curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$EVENTINDEXNAME/_mapping" -d ' @@ -54,3 +105,6 @@ curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$NOTIFICATIONINDEX } } }' + +echo +echo \ No newline at end of file diff --git a/apps/dhp-broker-application/src/main/resources/static/html/summary.html b/apps/dhp-broker-application/src/main/resources/static/html/summary.html index 5015069a..93a1a191 100644 --- a/apps/dhp-broker-application/src/main/resources/static/html/summary.html +++ b/apps/dhp-broker-application/src/main/resources/static/html/summary.html @@ -133,13 +133,9 @@ indexes - {{summary.elasticSearch.eventsIndexName}} - type - {{summary.elasticSearch.eventsIndexType}} + {{summary.elasticSearch.eventsIndexName}} - {{summary.elasticSearch.notificationsIndexName}} - type - {{summary.elasticSearch.notificationsIndexType}} + {{summary.elasticSearch.notificationsIndexName}} socket timeout