optimizations of es settings

This commit is contained in:
Michele Artini 2020-07-28 11:30:06 +02:00
parent b49ec4cc69
commit 81ed82683d
7 changed files with 69 additions and 44 deletions

View File

@ -7,7 +7,7 @@ import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
@Document(indexName = "#{elasticSearchProperties.eventsIndexName}", type = "#{elasticSearchProperties.eventsIndexType}")
@Document(indexName = "#{elasticSearchProperties.eventsIndexName}", shards = 48, replicas = 0, refreshInterval = "600s")
public class Event {
@Id
@ -37,8 +37,8 @@ public class Event {
public Event() {}
public Event(final String producerId, final String eventId, final String topic, final String payload, final Long creationDate, final Long expiryDate,
final boolean instantMessage,
final Map<String, Object> map) {
final boolean instantMessage,
final Map<String, Object> map) {
this.producerId = producerId;
this.eventId = eventId;
this.topic = topic;

View File

@ -11,7 +11,7 @@ import org.springframework.data.elasticsearch.annotations.FieldType;
import eu.dnetlib.lbs.subscriptions.Subscription;
@Document(indexName = "#{elasticSearchProperties.notificationsIndexName}", type = "#{elasticSearchProperties.notificationsIndexType}")
@Document(indexName = "#{elasticSearchProperties.notificationsIndexName}", shards = 48, replicas = 0, refreshInterval = "600s")
public class Notification {
@Id
@ -41,8 +41,8 @@ public class Notification {
public Notification() {}
public Notification(final String notificationId, final String subscriptionId, final String producerId, final String eventId, final String topic,
final String payload,
final Map<String, Object> map, final Long date) {
final String payload,
final Map<String, Object> map, final Long date) {
super();
this.notificationId = notificationId;
this.subscriptionId = subscriptionId;
@ -56,13 +56,13 @@ public class Notification {
public Notification(final Subscription s, final Event e) {
this(Notification.calculateId(s, e),
s.getSubscriptionId(),
e.getProducerId(),
e.getEventId(),
e.getTopic(),
e.getPayload(),
e.getMap(),
new Date().getTime());
s.getSubscriptionId(),
e.getProducerId(),
e.getEventId(),
e.getTopic(),
e.getPayload(),
e.getMap(),
new Date().getTime());
}
public String getNotificationId() {

View File

@ -110,8 +110,6 @@ public class SubscriptionEventMatcher implements Runnable {
.must(QueryBuilders.rangeQuery("creationDate").from(s.getLastNotificationDate() != null ? s.getLastNotificationDate().getTime() : 0))
.must(QueryBuilders.nestedQuery("map", mapQuery, ScoreMode.None)))
.withSearchType(SearchType.DEFAULT)
// .withIndices(elasticSearchProperties.getEventsIndexName())
// .withTypes(elasticSearchProperties.getEventsIndexType())
.withPageable(PageRequest.of(0, 10))
.build();

View File

@ -16,12 +16,8 @@ public class ElasticSearchProperties {
@NotNull
private String eventsIndexName;
@NotNull
private String eventsIndexType;
@NotNull
private String notificationsIndexName;
@NotNull
private String notificationsIndexType;
@NotNull
private long connectionTimeout;
@NotNull
private long socketTimeout;
@ -50,14 +46,6 @@ public class ElasticSearchProperties {
this.eventsIndexName = eventsIndexName;
}
public String getEventsIndexType() {
return this.eventsIndexType;
}
public void setEventsIndexType(final String eventsIndexType) {
this.eventsIndexType = eventsIndexType;
}
public String getNotificationsIndexName() {
return this.notificationsIndexName;
}
@ -66,14 +54,6 @@ public class ElasticSearchProperties {
this.notificationsIndexName = notificationsIndexName;
}
public String getNotificationsIndexType() {
return this.notificationsIndexType;
}
public void setNotificationsIndexType(final String notificationsIndexType) {
this.notificationsIndexType = notificationsIndexType;
}
public long getConnectionTimeout() {
return connectionTimeout;
}

View File

@ -27,11 +27,8 @@ lbs.elastic.clusterNodes = 127.0.0.1:9200
lbs.elastic.homepage = http://
lbs.elastic.socketTimeout = 30000
lbs.elastic.connectionTimeout = 30000
lbs.elastic.eventsIndexName = oa_events_beta
lbs.elastic.eventsIndexType = event
lbs.elastic.notificationsIndexName = oa_notifications_beta
lbs.elastic.notificationsIndexType = notification
lbs.mail.from = notifications-noreply@openaire.eu
lbs.mail.fromName = Literature Broker Service

View File

@ -4,6 +4,57 @@ INDEXBASEURL=http://localhost:9200
EVENTINDEXNAME=oa_events_beta
NOTIFICATIONINDEXNAME=oa_notifications_beta
echo
echo "Update settings of events index..."
curl -XPOST "$INDEXBASEURL/$EVENTINDEXNAME/_close"
echo
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$EVENTINDEXNAME/_settings" -d '
{
"index": {
"translog": {
"sync_interval": "15s",
"durability": "ASYNC"
},
"analysis": {
"analyzer": {
"analyzer_keyword": {
"filter": "lowercase",
"tokenizer": "keyword"
}
}
}
}
}'
echo
curl -XPOST "$INDEXBASEURL/$EVENTINDEXNAME/_open"
echo
echo
echo "Update settings of notifications index..."
curl -XPOST "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_close"
echo
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_settings" -d '
{
"index": {
"translog": {
"sync_interval": "15s",
"durability": "ASYNC"
},
"analysis": {
"analyzer": {
"analyzer_keyword": {
"filter": "lowercase",
"tokenizer": "keyword"
}
}
}
}
}'
echo
curl -XPOST "$INDEXBASEURL/$NOTIFICATIONINDEXNAME/_open"
echo
echo
echo "Fixing events index..."
curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$EVENTINDEXNAME/_mapping" -d '
@ -54,3 +105,6 @@ curl -XPUT -H 'Content-Type: application/json' "$INDEXBASEURL/$NOTIFICATIONINDEX
}
}
}'
echo
echo

View File

@ -133,13 +133,9 @@
</tr>
<tr>
<th class="col-xs-4" rowspan="2">indexes</th>
<td class="col-xs-4">{{summary.elasticSearch.eventsIndexName}}</td>
<th class="col-xs-1">type</th>
<td class="col-xs-3">{{summary.elasticSearch.eventsIndexType}}</td>
<td class="col-xs-8">{{summary.elasticSearch.eventsIndexName}}</td>
<tr>
<td class="col-xs-4">{{summary.elasticSearch.notificationsIndexName}}</td>
<th class="col-xs-1">type</th>
<td class="col-xs-3">{{summary.elasticSearch.notificationsIndexType}}</td>
<td class="col-xs-8">{{summary.elasticSearch.notificationsIndexName}}</td>
</tr>
<tr>
<th class="col-xs-4">socket timeout</th>