From 53750ff8184d359e185ef5ca2bf5abf8cf55660f Mon Sep 17 00:00:00 2001 From: "michele.artini" Date: Fri, 30 Jul 2021 14:12:09 +0200 Subject: [PATCH] new metrics --- .../broker/controllers/AjaxController.java | 16 ++------ .../src/main/resources/sql/totalEvents.sql | 1 - .../controllers/OpenairePublicController.java | 24 +++++++----- .../metrics/ValidOrganizationsMetric.java | 21 +++++++++++ .../repository/OrganizationRepository.java | 2 + .../eu/dnetlib/common/metrics/MetricInfo.java | 34 +++++++++++++++++ libs/dnet-broker-apps-common/pom.xml | 31 +++------------- .../elasticsearch/EventStatsManager.java | 4 -- .../TotalDatasourcesWithEventsMetric.java | 20 ++++++++++ ...talDatasourcesWithSubscriptionsMetric.java | 37 +++++++++++++++++++ .../common/metrics/TotalEventsMetric.java | 20 ++++++++++ .../metrics/TotalNotificationsMetric.java | 20 ++++++++++ .../metrics/TotalSubscribersMetric.java | 20 ++++++++++ .../metrics/TotalSubscriptionsMetric.java | 20 ++++++++++ .../stats/OpenaireDsStatRepository.java | 6 +++ .../subscriptions/SubscriptionRepository.java | 4 ++ 16 files changed, 227 insertions(+), 53 deletions(-) delete mode 100644 apps/dhp-broker-application/src/main/resources/sql/totalEvents.sql create mode 100644 apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/metrics/ValidOrganizationsMetric.java create mode 100644 libs/dnet-apps-common/src/main/java/eu/dnetlib/common/metrics/MetricInfo.java create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithEventsMetric.java create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithSubscriptionsMetric.java create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalEventsMetric.java create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalNotificationsMetric.java create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscribersMetric.java create mode 100644 libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscriptionsMetric.java diff --git a/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/AjaxController.java b/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/AjaxController.java index 2b6399a1..1242dd06 100644 --- a/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/AjaxController.java +++ b/apps/dhp-broker-application/src/main/java/eu/dnetlib/broker/controllers/AjaxController.java @@ -4,16 +4,15 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.io.IOUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import eu.dnetlib.broker.common.elasticsearch.EventRepository; import eu.dnetlib.broker.common.elasticsearch.NotificationRepository; +import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository; import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository; import eu.dnetlib.broker.common.topics.TopicTypeRepository; import eu.dnetlib.broker.controllers.objects.BufferStatus; @@ -57,7 +56,7 @@ public class AjaxController extends AbstractDnetController { private ThreadManager threadManager; @Autowired - private JdbcTemplate jdbcTemplate; + private OpenaireDsStatRepository openaireDsStatRepository; @Value("${lbs.elastic.homepage}") private String elasticSearchUiUrl; @@ -98,7 +97,7 @@ public class AjaxController extends AbstractDnetController { currentStatus.setThreads(threads); currentStatus.getTotals().put("topics", topicTypeRepo.count()); currentStatus.getTotals().put("events_es", eventRepository.count()); - currentStatus.getTotals().put("events_db", countEventsInTheDb()); + currentStatus.getTotals().put("events_db", openaireDsStatRepository.totalEvents()); currentStatus.getTotals().put("subscriptions", subscriptionRepo.count()); currentStatus.getTotals().put("notifications_es", notificationRepository.count()); @@ -106,15 +105,6 @@ public class AjaxController extends AbstractDnetController { } } - private Long countEventsInTheDb() { - try { - final String sql = IOUtils.toString(getClass().getResourceAsStream("/sql/totalEvents.sql")); - return jdbcTemplate.queryForObject(sql, Long.class); - } catch (final Exception e) { - return 0l; - } - } - @GetMapping("/resetCounters") public CurrentStatus resetCounters() { dispatcherManager.getDispatchers().forEach(NotificationDispatcher::resetCount); diff --git a/apps/dhp-broker-application/src/main/resources/sql/totalEvents.sql b/apps/dhp-broker-application/src/main/resources/sql/totalEvents.sql deleted file mode 100644 index 33bfd5ec..00000000 --- a/apps/dhp-broker-application/src/main/resources/sql/totalEvents.sql +++ /dev/null @@ -1 +0,0 @@ -select sum(size) from oa_datasource_stats; diff --git a/apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/OpenairePublicController.java b/apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/OpenairePublicController.java index 16663b29..c5ad736e 100644 --- a/apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/OpenairePublicController.java +++ b/apps/dhp-broker-public-application/src/main/java/eu/dnetlib/broker/oa/controllers/OpenairePublicController.java @@ -4,6 +4,7 @@ import java.io.BufferedReader; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -57,6 +58,7 @@ import eu.dnetlib.broker.common.elasticsearch.EventRepository; import eu.dnetlib.broker.common.elasticsearch.Notification; import eu.dnetlib.broker.common.elasticsearch.NotificationRepository; import eu.dnetlib.broker.common.properties.ElasticSearchProperties; +import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository; import eu.dnetlib.broker.common.subscriptions.Subscription; import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository; import eu.dnetlib.broker.objects.OaBrokerEventPayload; @@ -82,6 +84,9 @@ public class OpenairePublicController extends AbstractDnetController { @Autowired private SubscriptionRepository subscriptionRepo; + @Autowired + private OpenaireDsStatRepository openaireDsStatRepository; + @Autowired private ElasticSearchProperties props; @@ -154,7 +159,7 @@ public class OpenairePublicController extends AbstractDnetController { boolean first = true; - IOUtils.write("[\n", gzOut); + IOUtils.write("[\n", gzOut, StandardCharsets.UTF_8); ScrollPage page = null; @@ -165,13 +170,13 @@ public class OpenairePublicController extends AbstractDnetController { if (first) { first = false; } else { - IOUtils.write(",\n", gzOut); + IOUtils.write(",\n", gzOut, StandardCharsets.UTF_8); } - IOUtils.write(gson.toJson(msg), gzOut); + IOUtils.write(gson.toJson(msg), gzOut, StandardCharsets.UTF_8); } } while (!page.isCompleted()); - IOUtils.write("\n]\n", gzOut); + IOUtils.write("\n]\n", gzOut, StandardCharsets.UTF_8); gzOut.flush(); @@ -196,7 +201,7 @@ public class OpenairePublicController extends AbstractDnetController { final GZIPOutputStream gzOut = new GZIPOutputStream(out)) { boolean first = true; - IOUtils.write("[\n", gzOut); + IOUtils.write("[\n", gzOut, StandardCharsets.UTF_8); try { for (final FileStatus fileStatus : fs.listStatus(pathDir)) { @@ -223,7 +228,7 @@ public class OpenairePublicController extends AbstractDnetController { } catch (final FileNotFoundException e) { log.warn("File not found - " + e.getMessage()); } - IOUtils.write("\n]\n", gzOut); + IOUtils.write("\n]\n", gzOut, StandardCharsets.UTF_8); gzOut.flush(); } catch (final Throwable e) { log.error("Error accessing hdfs file", e); @@ -237,10 +242,10 @@ public class OpenairePublicController extends AbstractDnetController { if (first) { first = false; } else { - IOUtils.write(",\n", gzOut); + IOUtils.write(",\n", gzOut, StandardCharsets.UTF_8); } - IOUtils.write(line, gzOut); + IOUtils.write(line, gzOut, StandardCharsets.UTF_8); line = br.readLine(); } @@ -258,7 +263,8 @@ public class OpenairePublicController extends AbstractDnetController { private Map status() { final Map res = new LinkedHashMap<>(); res.put("n_subscriptions", subscriptionRepo.count()); - res.put("n_events", eventRepository.count()); + res.put("n_events_es", eventRepository.count()); + res.put("n_events_db", openaireDsStatRepository.totalEvents()); res.put("n_notifications", notificationRepository.count()); return res; } diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/metrics/ValidOrganizationsMetric.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/metrics/ValidOrganizationsMetric.java new file mode 100644 index 00000000..ccfd722b --- /dev/null +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/metrics/ValidOrganizationsMetric.java @@ -0,0 +1,21 @@ +package eu.dnetlib.organizations.metrics; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.common.metrics.MetricInfo; +import eu.dnetlib.organizations.repository.OrganizationRepository; +import eu.dnetlib.organizations.utils.OrganizationStatus; + +@Component("valid_organizations_total") +public class ValidOrganizationsMetric extends MetricInfo { + + @Autowired + private OrganizationRepository organizationRepository; + + @Override + public double obtainValue(final MetricInfo info) { + return organizationRepository.countByStatus(OrganizationStatus.approved.toString()); + } + +} diff --git a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java index cd9a50c7..6071b0d4 100644 --- a/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java +++ b/apps/dnet-orgs-database-application/src/main/java/eu/dnetlib/organizations/repository/OrganizationRepository.java @@ -26,4 +26,6 @@ public interface OrganizationRepository extends JpaRepository3.2.1 - - io.prometheus - simpleclient_spring_boot - ${prometheus.version} - - - org.springframework - spring-web - - + eu.dnetlib.dhp + dnet-apps-common + ${project.version} + provided - - io.prometheus - simpleclient_hotspot - ${prometheus.version} - - - io.prometheus - simpleclient_servlet - ${prometheus.version} - - - io.prometheus - simpleclient_spring_web - 0.3.0 - - + junit diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/EventStatsManager.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/EventStatsManager.java index 9ecb5856..9b0c42ac 100644 --- a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/EventStatsManager.java +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/elasticsearch/EventStatsManager.java @@ -3,8 +3,6 @@ package eu.dnetlib.broker.common.elasticsearch; import java.util.List; import java.util.stream.Collectors; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; import org.elasticsearch.action.search.SearchType; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -29,8 +27,6 @@ public class EventStatsManager { @Autowired private ElasticSearchProperties elasticSearchProperties; - private static final Log log = LogFactory.getLog(EventStatsManager.class); - public class BrowseEntry { private final String value; diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithEventsMetric.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithEventsMetric.java new file mode 100644 index 00000000..8f59d623 --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithEventsMetric.java @@ -0,0 +1,20 @@ +package eu.dnetlib.broker.common.metrics; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository; +import eu.dnetlib.common.metrics.MetricInfo; + +@Component("provide_broker_datasources_with_events") +public class TotalDatasourcesWithEventsMetric extends MetricInfo { + + @Autowired + private OpenaireDsStatRepository openaireDsStatRepository; + + @Override + public double obtainValue(final MetricInfo info) { + return openaireDsStatRepository.countDatasourcesWithEvents(); + } + +} diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithSubscriptionsMetric.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithSubscriptionsMetric.java new file mode 100644 index 00000000..9dff9986 --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalDatasourcesWithSubscriptionsMetric.java @@ -0,0 +1,37 @@ +package eu.dnetlib.broker.common.metrics; + +import java.util.List; +import java.util.stream.StreamSupport; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.broker.common.subscriptions.ConditionOperator; +import eu.dnetlib.broker.common.subscriptions.MapCondition; +import eu.dnetlib.broker.common.subscriptions.Subscription; +import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository; +import eu.dnetlib.common.metrics.MetricInfo; + +@Component("provide_broker_datasources_with_subscriptions") +public class TotalDatasourcesWithSubscriptionsMetric extends MetricInfo { + + @Autowired + private SubscriptionRepository subscriptionRepository; + + @Override + public double obtainValue(final MetricInfo info) { + return StreamSupport.stream(subscriptionRepository.findAll().spliterator(), false) + .map(Subscription::getConditionsAsList) + .flatMap(List::stream) + .filter(c -> c.getField().equals("targetDatasourceName")) + .filter(c -> c.getOperator() == ConditionOperator.EXACT) + .map(MapCondition::getListParams) + .filter(l -> !l.isEmpty()) + .map(l -> l.get(0).getValue()) + .filter(StringUtils::isNotBlank) + .distinct() + .count(); + } + +} diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalEventsMetric.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalEventsMetric.java new file mode 100644 index 00000000..927b8675 --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalEventsMetric.java @@ -0,0 +1,20 @@ +package eu.dnetlib.broker.common.metrics; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.broker.common.stats.OpenaireDsStatRepository; +import eu.dnetlib.common.metrics.MetricInfo; + +@Component("provide_broker_events") +public class TotalEventsMetric extends MetricInfo { + + @Autowired + private OpenaireDsStatRepository openaireDsStatRepository; + + @Override + public double obtainValue(final MetricInfo info) { + return openaireDsStatRepository.totalEvents(); + } + +} diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalNotificationsMetric.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalNotificationsMetric.java new file mode 100644 index 00000000..888805a1 --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalNotificationsMetric.java @@ -0,0 +1,20 @@ +package eu.dnetlib.broker.common.metrics; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.broker.common.elasticsearch.NotificationRepository; +import eu.dnetlib.common.metrics.MetricInfo; + +@Component("provide_broker_notifications") +public class TotalNotificationsMetric extends MetricInfo { + + @Autowired + private NotificationRepository notificationRepository; + + @Override + public double obtainValue(final MetricInfo info) { + return notificationRepository.count(); + } + +} diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscribersMetric.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscribersMetric.java new file mode 100644 index 00000000..f1f9f426 --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscribersMetric.java @@ -0,0 +1,20 @@ +package eu.dnetlib.broker.common.metrics; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository; +import eu.dnetlib.common.metrics.MetricInfo; + +@Component("provide_broker_subscribers") +public class TotalSubscribersMetric extends MetricInfo { + + @Autowired + private SubscriptionRepository subscriptionRepository; + + @Override + public double obtainValue(final MetricInfo info) { + return subscriptionRepository.countSubscribers(); + } + +} diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscriptionsMetric.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscriptionsMetric.java new file mode 100644 index 00000000..9a9ee34d --- /dev/null +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/metrics/TotalSubscriptionsMetric.java @@ -0,0 +1,20 @@ +package eu.dnetlib.broker.common.metrics; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import eu.dnetlib.broker.common.subscriptions.SubscriptionRepository; +import eu.dnetlib.common.metrics.MetricInfo; + +@Component("provide_broker_subscriptions") +public class TotalSubscriptionsMetric extends MetricInfo { + + @Autowired + private SubscriptionRepository subscriptionRepository; + + @Override + public double obtainValue(final MetricInfo info) { + return subscriptionRepository.count(); + } + +} diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java index 95c7ce86..84b34d97 100644 --- a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/stats/OpenaireDsStatRepository.java @@ -1,7 +1,13 @@ package eu.dnetlib.broker.common.stats; +import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; public interface OpenaireDsStatRepository extends CrudRepository { + @Query(value = "select sum(size) from oa_datasource_stats", nativeQuery = true) + long totalEvents(); + + @Query(value = "select count(distinct name) from oa_datasource_stats where size > 0", nativeQuery = true) + long countDatasourcesWithEvents(); } diff --git a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/subscriptions/SubscriptionRepository.java b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/subscriptions/SubscriptionRepository.java index 42d61f85..0e62e053 100644 --- a/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/subscriptions/SubscriptionRepository.java +++ b/libs/dnet-broker-apps-common/src/main/java/eu/dnetlib/broker/common/subscriptions/SubscriptionRepository.java @@ -1,5 +1,6 @@ package eu.dnetlib.broker.common.subscriptions; +import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.CrudRepository; import org.springframework.stereotype.Repository; @@ -10,4 +11,7 @@ public interface SubscriptionRepository extends CrudRepository findBySubscriber(String email); + @Query(value = "select count(distinct subscriber) from subscriptions", nativeQuery = true) + long countSubscribers(); + }