package eu.eudat.service.metrics; import eu.eudat.commons.enums.*; import eu.eudat.commons.metrics.MetricLabels; import eu.eudat.commons.metrics.MetricNames; import eu.eudat.data.TenantEntityManager; import eu.eudat.query.*; import eu.eudat.service.keycloak.MyKeycloakAdminRestApi; import gr.cite.commons.web.keycloak.api.configuration.KeycloakClientProperties; import gr.cite.tools.data.query.QueryFactory; import gr.cite.tools.logging.LoggerService; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.Gauge; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.management.InvalidApplicationException; import java.util.Map; import java.util.stream.Collectors; import java.util.stream.Stream; @Service public class MetricsServiceImpl implements MetricsService { private final PrometheusMeterRegistry registry; private final QueryFactory queryFactory; private final TenantEntityManager entityManager; private final MyKeycloakAdminRestApi keycloakAdminRestApi; private final UpdateMetricsTaskProperties _config; private final KeycloakClientProperties _keycloakConfig; private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(MetricsServiceImpl.class)); public MetricsServiceImpl( PrometheusMeterRegistry registry, QueryFactory queryFactory, TenantEntityManager entityManager, MyKeycloakAdminRestApi keycloakAdminRestApi, UpdateMetricsTaskProperties config, KeycloakClientProperties keycloakConfig) { this.registry = registry; this.queryFactory = queryFactory; this.entityManager = entityManager; this.keycloakAdminRestApi = keycloakAdminRestApi; this._config = config; this._keycloakConfig = keycloakConfig; } @Override public void calculate(Map gauges) throws InvalidApplicationException { try { this.entityManager.disableTenantFilters(); this.setGaugeValue(gauges, MetricNames.DMP, calculateDraftDmps(false), MetricLabels.DRAFT); this.setGaugeValue(gauges, MetricNames.DMP, calculateFinalizedDmps(false), MetricLabels.FINALIZED); this.setGaugeValue(gauges, MetricNames.DMP, calculatePublishedDmps(false), MetricLabels.PUBLISHED); this.setGaugeValue(gauges, MetricNames.DMP, calculateDoiedDmps(false), MetricLabels.DOIED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateDraftDmps(true), MetricLabels.DRAFT); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateFinalizedDmps(true), MetricLabels.FINALIZED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculatePublishedDmps(true), MetricLabels.PUBLISHED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateDoiedDmps(true), MetricLabels.DOIED); this.setGaugeValue(gauges, MetricNames.DMP_WITH_GRANT, calculateDraftDmpsWithGrant(false), MetricLabels.DRAFT); this.setGaugeValue(gauges, MetricNames.DMP_WITH_GRANT, calculateFinalizedDmpsWithGrant(false), MetricLabels.FINALIZED); this.setGaugeValue(gauges, MetricNames.DMP_WITH_GRANT, calculatePublishedDmpsWithGrant(false), MetricLabels.PUBLISHED); this.setGaugeValue(gauges, MetricNames.DMP_WITH_GRANT, calculateDoiedDmpsWithGrant(false), MetricLabels.DOIED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP_WITH_GRANT, calculateDraftDmpsWithGrant(true), MetricLabels.DRAFT); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP_WITH_GRANT, calculateFinalizedDmpsWithGrant(true), MetricLabels.FINALIZED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP_WITH_GRANT, calculatePublishedDmpsWithGrant(true), MetricLabels.PUBLISHED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP_WITH_GRANT, calculateDoiedDmpsWithGrant(true), MetricLabels.DOIED); this.setGaugeValue(gauges, MetricNames.FUNDERS, calculateAllFunders(false), null); this.setGaugeValue(gauges, MetricNames.GRANTS, calculateAllGrants(false), null); this.setGaugeValue(gauges, MetricNames.PROJECTS, calculateAllProjects(false), null); this.setGaugeValue(gauges, MetricNames.RESEARCHERS, calculateAllResearchers(false), null); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.FUNDERS, calculateAllFunders(true), null); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.GRANTS, calculateAllGrants(true), null); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.PROJECTS, calculateAllProjects(true), null); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.RESEARCHERS, calculateAllResearchers(true), null); this.setGaugeValue(gauges, MetricNames.DATASET, calculateDraftDatasets(false), MetricLabels.DRAFT); this.setGaugeValue(gauges, MetricNames.DATASET, calculateFinalizedDatasets(false), MetricLabels.FINALIZED); this.setGaugeValue(gauges, MetricNames.DATASET, calculatePublishedDatasets(false), MetricLabels.PUBLISHED); this.setGaugeValue(gauges, MetricNames.DATASET, calculateDoiedDatasets(false), MetricLabels.DOIED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET, calculateDraftDatasets(true), MetricLabels.DRAFT); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET, calculateFinalizedDatasets(true), MetricLabels.FINALIZED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET, calculatePublishedDatasets(true), MetricLabels.PUBLISHED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET, calculateDoiedDatasets(true), MetricLabels.DOIED); this.setGaugeValue(gauges, MetricNames.DATASET_TEMPLATE, calculateDraftTemplates(false), MetricLabels.DRAFT); this.setGaugeValue(gauges, MetricNames.DATASET_TEMPLATE, calculateFinalizedTemplates(false), MetricLabels.ACTIVE); this.setGaugeValue(gauges, MetricNames.DATASET_TEMPLATE, calculateUsedTemplates(false), MetricLabels.USED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, calculateDraftTemplates(true), MetricLabels.DRAFT); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, calculateFinalizedTemplates(true), MetricLabels.ACTIVE); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, calculateUsedTemplates(true), MetricLabels.USED); this.setGaugeValue(gauges, MetricNames.LANGUAGES, calculateLanguages(), null); this.setGaugeValue(gauges, MetricNames.INSTALLATIONS, 1d, null); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.INSTALLATIONS, 1d, null); this.setGaugeValue(gauges, MetricNames.USERS, calculateActiveUsers(), MetricLabels.LOGGEDIN); this.setGaugeValue(gauges, MetricNames.USERS, calculateAllUsers(), MetricLabels.TOTAL); } catch (Exception e) { logger.error(e.getMessage(), e); } finally { this.entityManager.enableTenantFilters(); } } @Override public Map gaugesBuild() { registry.clear(); return Stream.of(new Object[][]{ {MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())}, {MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())}, {MetricNames.PROJECTS, Gauge.build().name(MetricNames.PROJECTS).help("Number of registered Projects").register(registry.getPrometheusRegistry())}, {MetricNames.RESEARCHERS, Gauge.build().name(MetricNames.RESEARCHERS).help("Number of Collaborators/Researchers").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.PROJECTS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.PROJECTS).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.RESEARCHERS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.RESEARCHERS).help("Number of Collaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())}, {MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())}, {MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())}, {MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS_PREFIX + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())}, }).collect(Collectors.toMap(data -> (String) data[0], data -> (Gauge) data[1])); } private double calculateDraftDmps(boolean forNexus) { DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Draft).isActive(IsActive.Active); if (forNexus) dmpQuery.after(_config.getNexusDate()); return dmpQuery.count(); } private double calculateFinalizedDmps(boolean forNexus) { DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).isActive(IsActive.Active); if (forNexus) dmpQuery.after(_config.getNexusDate()); return dmpQuery.count(); } private double calculatePublishedDmps(boolean forNexus) { DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).accessTypes(DmpAccessType.Public).isActive(IsActive.Active); if (forNexus) dmpQuery.after(_config.getNexusDate()); return dmpQuery.count(); } private double calculateDoiedDmps(boolean forNexus) { DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).isActive(IsActive.Active); if (forNexus) dmpQuery.after(_config.getNexusDate()); EntityDoiQuery entityDoiQuery = this.queryFactory.query(EntityDoiQuery.class).types(EntityType.DMP).isActive(IsActive.Active); dmpQuery.entityDoiSubQuery(entityDoiQuery); dmpQuery.setDistinct(true); return dmpQuery.count(); } private double calculateDraftDmpsWithGrant(boolean forNexus) { DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Draft).isActive(IsActive.Active); if (forNexus) dmpQuery.after(_config.getNexusDate()); ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); DmpReferenceQuery dmpReferenceQuery = this.queryFactory.query(DmpReferenceQuery.class).referenceSubQuery(referenceQuery).isActives(IsActive.Active); dmpQuery.dmpReferenceSubQuery(dmpReferenceQuery); return dmpQuery.count(); } private double calculateFinalizedDmpsWithGrant(boolean forNexus) { DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).isActive(IsActive.Active); if (forNexus) dmpQuery.after(_config.getNexusDate()); ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); DmpReferenceQuery dmpReferenceQuery = this.queryFactory.query(DmpReferenceQuery.class).referenceSubQuery(referenceQuery).isActives(IsActive.Active); dmpQuery.dmpReferenceSubQuery(dmpReferenceQuery); return dmpQuery.count(); } private double calculatePublishedDmpsWithGrant(boolean forNexus) { DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).accessTypes(DmpAccessType.Public).isActive(IsActive.Active); if (forNexus) dmpQuery.after(_config.getNexusDate()); ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); DmpReferenceQuery dmpReferenceQuery = this.queryFactory.query(DmpReferenceQuery.class).referenceSubQuery(referenceQuery).isActives(IsActive.Active); dmpQuery.dmpReferenceSubQuery(dmpReferenceQuery); return dmpQuery.count(); } private double calculateDoiedDmpsWithGrant(boolean forNexus) { DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).isActive(IsActive.Active); if (forNexus) dmpQuery.after(_config.getNexusDate()); ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); DmpReferenceQuery dmpReferenceQuery = this.queryFactory.query(DmpReferenceQuery.class).referenceSubQuery(referenceQuery).isActives(IsActive.Active); dmpQuery.dmpReferenceSubQuery(dmpReferenceQuery); EntityDoiQuery entityDoiQuery = this.queryFactory.query(EntityDoiQuery.class).isActive(IsActive.Active); dmpQuery.entityDoiSubQuery(entityDoiQuery); return dmpQuery.count(); } private double calculateAllFunders(boolean forNexus) { ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getFunderIds()).isActive(IsActive.Active); if (forNexus) referenceQuery.after(_config.getNexusDate()); return referenceQuery.count(); } private double calculateAllGrants(boolean forNexus) { ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); if (forNexus) referenceQuery.after(_config.getNexusDate()); return referenceQuery.count(); } private double calculateAllProjects(boolean forNexus) { ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getProjectIds()).isActive(IsActive.Active); if (forNexus) referenceQuery.after(_config.getNexusDate()); return referenceQuery.count(); } private double calculateAllResearchers(boolean forNexus) { ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getResearcherIds()).isActive(IsActive.Active); if (forNexus) referenceQuery.after(_config.getNexusDate()); return referenceQuery.count(); } private double calculateDraftDatasets(boolean forNexus) { DescriptionQuery descriptionQuery = this.queryFactory.query(DescriptionQuery.class).statuses(DescriptionStatus.Draft).isActive(IsActive.Active); if (forNexus) descriptionQuery.createdAfter(_config.getNexusDate()); return descriptionQuery.count(); } private double calculateFinalizedDatasets(boolean forNexus) { DescriptionQuery descriptionQuery = this.queryFactory.query(DescriptionQuery.class).statuses(DescriptionStatus.Finalized).isActive(IsActive.Active); if (forNexus) descriptionQuery.createdAfter(_config.getNexusDate()); return descriptionQuery.count(); } private double calculatePublishedDatasets(boolean forNexus) { DescriptionQuery descriptionQuery = this.queryFactory.query(DescriptionQuery.class).statuses(DescriptionStatus.Finalized).isActive(IsActive.Active); if (forNexus) descriptionQuery.createdAfter(_config.getNexusDate()); DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).isActive(IsActive.Active).statuses(DmpStatus.Finalized).accessTypes(DmpAccessType.Public); descriptionQuery.dmpSubQuery(dmpQuery); return descriptionQuery.count(); } private double calculateDoiedDatasets(boolean forNexus) { return 0; } private double calculateDraftTemplates(boolean forNexus) { DescriptionTemplateQuery descriptionTemplateQuery = this.queryFactory.query(DescriptionTemplateQuery.class).statuses(DescriptionTemplateStatus.Draft).isActive(IsActive.Active); if (forNexus) descriptionTemplateQuery.after(_config.getNexusDate()); return descriptionTemplateQuery.count(); } private double calculateFinalizedTemplates(boolean forNexus) { DescriptionTemplateQuery descriptionTemplateQuery = this.queryFactory.query(DescriptionTemplateQuery.class).statuses(DescriptionTemplateStatus.Finalized).isActive(IsActive.Active); if (forNexus) descriptionTemplateQuery.after(_config.getNexusDate()); return descriptionTemplateQuery.count(); } private double calculateUsedTemplates(boolean forNexus) { DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery = this.queryFactory.query(DmpDescriptionTemplateQuery.class).isActive(IsActive.Active); dmpDescriptionTemplateQuery.setDistinct(true); if (forNexus) dmpDescriptionTemplateQuery.after(_config.getNexusDate()); return dmpDescriptionTemplateQuery.count(); } private double calculateActiveUsers() { double result = -1; try { result = this.keycloakAdminRestApi.users().getUserSessionsCountByClientId(_config.getUsersLoginClient()); } catch (Exception e) { logger.error(e.getMessage(), e); } return result; } private double calculateAllUsers() { UserQuery userQuery = this.queryFactory.query(UserQuery.class).isActive(IsActive.Active); return userQuery.count(); } private double calculateLanguages() { LanguageQuery languageQuery = this.queryFactory.query(LanguageQuery.class).isActive(IsActive.Active); return languageQuery.count(); } private void setGaugeValue(Map gauges, String name, Double amount, String label) { if (label != null) { gauges.get(name).labels(label).set(amount); } else { gauges.get(name).set(amount); } } }