D-Net
/
dnet-applications
16
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

metrics cache

This commit is contained in:
Michele Artini 2021-07-30 16:17:20 +02:00
parent 53750ff818
commit 91fccbae7f
12 changed files with 120 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/LiteratureBrokerServiceConfiguration.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.broker;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
@ -18,6 +19,7 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
@Configuration
@EnableSwagger2
@EnableCaching
@EnableScheduling
@EnableTransactionManagement
@EnableElasticsearchRepositories(basePackageClasses = {

2
apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/BrokerConfiguration.java
View File

@ -2,6 +2,7 @@ package eu.dnetlib.broker;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.client.ClientConfiguration;
@ -18,6 +19,7 @@ import springfox.documentation.swagger2.annotations.EnableSwagger2;
@Configuration
@EnableSwagger2
@EnableCaching
@EnableScheduling
@EnableTransactionManagement
@EnableElasticsearchRepositories(basePackageClasses = {

1
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/controller/UserController.java
View File

@ -31,6 +31,7 @@ public class UserController extends AbstractDnetController {
@Autowired
private UserViewRepository userViewRepository;
@Autowired
private DatabaseUtils dbUtils;

2
apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/metrics/ValidOrganizationsMetric.java
View File

@ -14,7 +14,7 @@ public class ValidOrganizationsMetric extends MetricInfo {
private OrganizationRepository organizationRepository;
@Override
public double obtainValue(final MetricInfo info) {
public double obtainValue() {
return organizationRepository.countByStatus(OrganizationStatus.approved.toString());
}

8
libs/dnet-apps-common/src/main/java/eu/dnetlib/common/metrics/MetricInfo.java
View File

@ -14,21 +14,21 @@ public abstract class MetricInfo implements BeanNameAware {
private String beanName;
abstract public double obtainValue(MetricInfo info);
abstract public double obtainValue();
public String getBeanName() {
return beanName;
}
@Override
public final void setBeanName(final String beanName) {
public void setBeanName(final String beanName) {
this.beanName = beanName;
}
@PostConstruct
public final void register() {
public void register() {
log.info("Prometheus - new metric registered: " + getBeanName());
Metrics.gauge(getBeanName(), this, this::obtainValue);
Metrics.gauge(getBeanName(), this, o -> obtainValue());
}
}

92
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/MetricsCacheUtils.java Normal file
View File

@ -0,0 +1,92 @@
package eu.dnetlib.broker.common.metrics;
import java.util.List;
import java.util.stream.StreamSupport;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.cache.annotation.CacheEvict;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository;
import eu.dnetlib.broker.common.subscriptions.ConditionOperator;
import eu.dnetlib.broker.common.subscriptions.MapCondition;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
@Component
public class MetricsCacheUtils {
public static final String metrics_cache = "broker_metrics_cache";
private static final long interval = 12 * 60 * 60 * 1000; // 12 hours
private static final Log log = LogFactory.getLog(MetricsCacheUtils.class);
@Autowired
private SubscriptionRepository subscriptionRepository;
@Autowired
private OpenaireDsStatRepository openaireDsStatRepository;
@Autowired
private NotificationRepository notificationRepository;
@Scheduled(fixedDelay = interval, initialDelay = 500)
@CacheEvict(allEntries = true, value = metrics_cache)
public void reportCacheEvict() {
log.debug("Flush Cache: " + metrics_cache);
}
@Cacheable(value = MetricsCacheUtils.metrics_cache, key = "#root.methodName")
public double totalEvents() {
log.debug("Call to cached method: totalEvents()");
return openaireDsStatRepository.totalEvents();
}
@Cacheable(value = MetricsCacheUtils.metrics_cache, key = "#root.methodName")
public double countDatasourcesWithEvents() {
log.debug("Call to cached method: countDatasourcesWithEvents()");
return openaireDsStatRepository.countDatasourcesWithEvents();
}
@Cacheable(value = MetricsCacheUtils.metrics_cache, key = "#root.methodName")
public double countNotifications() {
log.debug("Call to cached method: countNotifications()");
return notificationRepository.count();
}
@Cacheable(value = MetricsCacheUtils.metrics_cache, key = "#root.methodName")
public double countSubscriptions() {
log.debug("Call to cached method: countSubscriptions()");
return subscriptionRepository.count();
}
@Cacheable(value = MetricsCacheUtils.metrics_cache, key = "#root.methodName")
public double countSubscribers() {
log.debug("Call to cached method: countSubscribers()");
return subscriptionRepository.countSubscribers();
}
@Cacheable(value = MetricsCacheUtils.metrics_cache, key = "#root.methodName")
public double countDatasourcesWithSubscriptions() {
log.debug("Call to cached method: countDatasourcesWithSubscriptions()");
return StreamSupport.stream(subscriptionRepository.findAll().spliterator(), false)
.map(Subscription::getConditionsAsList)
.flatMap(List::stream)
.filter(c -> c.getField().equals("targetDatasourceName"))
.filter(c -> c.getOperator() == ConditionOperator.EXACT)
.map(MapCondition::getListParams)
.filter(l -> !l.isEmpty())
.map(l -> l.get(0).getValue())
.filter(StringUtils::isNotBlank)
.distinct()
.count();
}
}

7
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithEventsMetric.java
View File

@ -3,18 +3,17 @@ package eu.dnetlib.broker.common.metrics;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository;
import eu.dnetlib.common.metrics.MetricInfo;
@Component("provide_broker_datasources_with_events")
public class TotalDatasourcesWithEventsMetric extends MetricInfo {
@Autowired
private OpenaireDsStatRepository openaireDsStatRepository;
private MetricsCacheUtils metricsCacheUtils;
@Override
public double obtainValue(final MetricInfo info) {
return openaireDsStatRepository.countDatasourcesWithEvents();
public double obtainValue() {
return metricsCacheUtils.countDatasourcesWithEvents();
}
}

24
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithSubscriptionsMetric.java
View File

@ -1,37 +1,19 @@
package eu.dnetlib.broker.common.metrics;
import java.util.List;
import java.util.stream.StreamSupport;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.broker.common.subscriptions.ConditionOperator;
import eu.dnetlib.broker.common.subscriptions.MapCondition;
import eu.dnetlib.broker.common.subscriptions.Subscription;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.common.metrics.MetricInfo;
@Component("provide_broker_datasources_with_subscriptions")
public class TotalDatasourcesWithSubscriptionsMetric extends MetricInfo {
@Autowired
private SubscriptionRepository subscriptionRepository;
private MetricsCacheUtils metricsCacheUtils;
@Override
public double obtainValue(final MetricInfo info) {
return StreamSupport.stream(subscriptionRepository.findAll().spliterator(), false)
.map(Subscription::getConditionsAsList)
.flatMap(List::stream)
.filter(c -> c.getField().equals("targetDatasourceName"))
.filter(c -> c.getOperator() == ConditionOperator.EXACT)
.map(MapCondition::getListParams)
.filter(l -> !l.isEmpty())
.map(l -> l.get(0).getValue())
.filter(StringUtils::isNotBlank)
.distinct()
.count();
public double obtainValue() {
return metricsCacheUtils.countDatasourcesWithSubscriptions();
}
}

7
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalEventsMetric.java
View File

@ -3,18 +3,17 @@ package eu.dnetlib.broker.common.metrics;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository;
import eu.dnetlib.common.metrics.MetricInfo;
@Component("provide_broker_events")
public class TotalEventsMetric extends MetricInfo {
@Autowired
private OpenaireDsStatRepository openaireDsStatRepository;
private MetricsCacheUtils metricsCacheUtils;
@Override
public double obtainValue(final MetricInfo info) {
return openaireDsStatRepository.totalEvents();
public double obtainValue() {
return metricsCacheUtils.totalEvents();
}
}

7
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalNotificationsMetric.java
View File

@ -3,18 +3,17 @@ package eu.dnetlib.broker.common.metrics;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.broker.common.elasticsearch.NotificationRepository;
import eu.dnetlib.common.metrics.MetricInfo;
@Component("provide_broker_notifications")
public class TotalNotificationsMetric extends MetricInfo {
@Autowired
private NotificationRepository notificationRepository;
private MetricsCacheUtils metricsCacheUtils;
@Override
public double obtainValue(final MetricInfo info) {
return notificationRepository.count();
public double obtainValue() {
return metricsCacheUtils.countNotifications();
}
}

7
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscribersMetric.java
View File

@ -3,18 +3,17 @@ package eu.dnetlib.broker.common.metrics;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.common.metrics.MetricInfo;
@Component("provide_broker_subscribers")
public class TotalSubscribersMetric extends MetricInfo {
@Autowired
private SubscriptionRepository subscriptionRepository;
private MetricsCacheUtils metricsCacheUtils;
@Override
public double obtainValue(final MetricInfo info) {
return subscriptionRepository.countSubscribers();
public double obtainValue() {
return metricsCacheUtils.countSubscribers();
}
}

7
libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscriptionsMetric.java
View File

@ -3,18 +3,17 @@ package eu.dnetlib.broker.common.metrics;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository;
import eu.dnetlib.common.metrics.MetricInfo;
@Component("provide_broker_subscriptions")
public class TotalSubscriptionsMetric extends MetricInfo {
@Autowired
private SubscriptionRepository subscriptionRepository;
private MetricsCacheUtils metricsCacheUtils;
@Override
public double obtainValue(final MetricInfo info) {
return subscriptionRepository.count();
public double obtainValue() {
return metricsCacheUtils.countSubscriptions();
}
}