From c477eb5460bd908bd535f365ee2bb9a72ed89632 Mon Sep 17 00:00:00 2001 From: Thomas Georgios Giannos Date: Fri, 12 Apr 2024 16:51:44 +0300 Subject: [PATCH] Adding back metrics for users, languages, installations, dmps with grants for prometheus --- .../main/java/eu/eudat/query/DmpQuery.java | 27 ++++- .../eudat/service/keycloak/MyUsersModule.java | 13 ++- .../service/metrics/MetricsServiceImpl.java | 101 +++++++++++++++++- 3 files changed, 129 insertions(+), 12 deletions(-) diff --git a/dmp-backend/core/src/main/java/eu/eudat/query/DmpQuery.java b/dmp-backend/core/src/main/java/eu/eudat/query/DmpQuery.java index 1345177dc..0b70cd869 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/query/DmpQuery.java +++ b/dmp-backend/core/src/main/java/eu/eudat/query/DmpQuery.java @@ -2,13 +2,11 @@ package eu.eudat.query; import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.authorization.Permission; -import eu.eudat.commons.enums.DmpAccessType; -import eu.eudat.commons.enums.DmpStatus; -import eu.eudat.commons.enums.DmpVersionStatus; -import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.*; import eu.eudat.commons.scope.user.UserScope; import eu.eudat.data.DmpDescriptionTemplateEntity; import eu.eudat.data.DmpEntity; +import eu.eudat.data.DmpReferenceEntity; import eu.eudat.data.DmpUserEntity; import eu.eudat.model.Dmp; import eu.eudat.model.PublicDmp; @@ -57,6 +55,10 @@ public class DmpQuery extends QueryBase { private DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery; + private DmpReferenceQuery dmpReferenceQuery; + + private EntityDoiQuery entityDoiQuery; + private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); private final UserScope userScope; @@ -226,6 +228,17 @@ public class DmpQuery extends QueryBase { return this; } + public DmpQuery dmpReferenceSubQuery(DmpReferenceQuery subQuery) { + this.dmpReferenceQuery = subQuery; + return this; + } + + public DmpQuery entityDoiSubQuery(EntityDoiQuery subQuery) { + this.entityDoiQuery = subQuery; + this.entityDoiQuery.types(EntityType.DMP); + return this; + } + public DmpQuery authorize(EnumSet values) { this.authorize = values; return this; @@ -345,6 +358,12 @@ public class DmpQuery extends QueryBase { QueryContext subQuery = this.applySubQuery(this.dmpDescriptionTemplateQuery, queryContext, UUID.class, dmpDescriptionTemplateEntityRoot -> dmpDescriptionTemplateEntityRoot.get(DmpDescriptionTemplateEntity._dmpId)); predicates.add(queryContext.CriteriaBuilder.in(queryContext.Root.get(DmpEntity._id)).value(subQuery.Query)); } + + if (this.dmpReferenceQuery != null) { + QueryContext subQuery = this.applySubQuery(this.dmpReferenceQuery, queryContext, UUID.class, dmpReferenceEntityRoot -> dmpReferenceEntityRoot.get(DmpReferenceEntity._dmpId)); + predicates.add(queryContext.CriteriaBuilder.in(queryContext.Root.get(DmpEntity._id)).value(subQuery.Query)); + } + if (!predicates.isEmpty()) { Predicate[] predicatesArray = predicates.toArray(new Predicate[0]); return queryContext.CriteriaBuilder.and(predicatesArray); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/keycloak/MyUsersModule.java b/dmp-backend/core/src/main/java/eu/eudat/service/keycloak/MyUsersModule.java index 4f3da992a..bc782ee73 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/keycloak/MyUsersModule.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/keycloak/MyUsersModule.java @@ -7,16 +7,13 @@ import org.keycloak.representations.idm.UserRepresentation; import java.util.List; public class MyUsersModule { + private final RealmResource realm; MyUsersModule(RealmResource realm) { this.realm = realm; } - public List getUsers() { - return this.realm.users().list(); - } - public UserRepresentation findUserById(String id) { return this.realm.users().get(id).toRepresentation(); } @@ -45,6 +42,14 @@ public class MyUsersModule { return this.realm.users().get(userId).groups(); } + public double getUserSessionsCountByClientId(String clientId) { + return this.realm.clients().get(clientId).getUserSessions(0, Integer.MAX_VALUE).size(); + } + + public double getAllUsersCount() { + return this.realm.users().count(); + } + public void updateUser(String userId, UserRepresentation user) { UserRepresentation existing = this.realm.users().get(userId).toRepresentation(); existing.setFirstName(user.getFirstName()); diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/metrics/MetricsServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/metrics/MetricsServiceImpl.java index 25e525c5c..20590b6ce 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/metrics/MetricsServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/metrics/MetricsServiceImpl.java @@ -3,15 +3,24 @@ package eu.eudat.service.metrics; import eu.eudat.commons.enums.*; import eu.eudat.commons.metrics.MetricLabels; import eu.eudat.commons.metrics.MetricNames; +import eu.eudat.data.DmpEntity; import eu.eudat.data.TenantEntityManager; +import eu.eudat.model.Dmp; import eu.eudat.query.*; +import eu.eudat.service.keycloak.KeycloakService; +import eu.eudat.service.keycloak.MyKeycloakAdminRestApi; +import gr.cite.commons.web.keycloak.api.configuration.KeycloakClientProperties; import gr.cite.tools.data.query.QueryFactory; +import gr.cite.tools.fieldset.BaseFieldSet; +import gr.cite.tools.fieldset.FieldSet; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.Gauge; import org.springframework.stereotype.Service; import javax.management.InvalidApplicationException; +import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -24,13 +33,24 @@ public class MetricsServiceImpl implements MetricsService { private final TenantEntityManager entityManager; + private final MyKeycloakAdminRestApi keycloakAdminRestApi; + private final UpdateMetricsTaskProperties _config; - public MetricsServiceImpl(PrometheusMeterRegistry registry, QueryFactory queryFactory, TenantEntityManager entityManager, UpdateMetricsTaskProperties config) { + private final KeycloakClientProperties _keycloakConfig; + + public MetricsServiceImpl( + PrometheusMeterRegistry registry, + QueryFactory queryFactory, + TenantEntityManager entityManager, + MyKeycloakAdminRestApi keycloakAdminRestApi, + UpdateMetricsTaskProperties config, KeycloakClientProperties keycloakConfig) { this.registry = registry; this.queryFactory = queryFactory; this.entityManager = entityManager; + this.keycloakAdminRestApi = keycloakAdminRestApi; this._config = config; + this._keycloakConfig = keycloakConfig; } @Override @@ -48,6 +68,16 @@ public class MetricsServiceImpl implements MetricsService { this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculatePublishedDmps(true), MetricLabels.PUBLISHED); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateDoiedDmps(true), MetricLabels.DOIED); + this.setGaugeValue(gauges, MetricNames.DMP, calculateDraftDmpsWithGrant(false), MetricLabels.DRAFT); + this.setGaugeValue(gauges, MetricNames.DMP, calculateFinalizedDmpsWithGrant(false), MetricLabels.FINALIZED); + this.setGaugeValue(gauges, MetricNames.DMP, calculatePublishedDmpsWithGrant(false), MetricLabels.PUBLISHED); + this.setGaugeValue(gauges, MetricNames.DMP, calculateDoiedDmpsWithGrant(false), MetricLabels.DOIED); + + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateDraftDmpsWithGrant(true), MetricLabels.DRAFT); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateFinalizedDmpsWithGrant(true), MetricLabels.FINALIZED); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculatePublishedDmpsWithGrant(true), MetricLabels.PUBLISHED); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateDoiedDmpsWithGrant(true), MetricLabels.DOIED); + this.setGaugeValue(gauges, MetricNames.FUNDERS, calculateAllFunders(false), null); this.setGaugeValue(gauges, MetricNames.GRANTS, calculateAllGrants(false), null); this.setGaugeValue(gauges, MetricNames.PROJECTS, calculateAllProjects(false), null); @@ -76,7 +106,13 @@ public class MetricsServiceImpl implements MetricsService { this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, calculateFinalizedTemplates(true), MetricLabels.ACTIVE); this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, calculateUsedTemplates(true), MetricLabels.USED); + this.setGaugeValue(gauges, MetricNames.USERS, calculateActiveUsers(), MetricLabels.LOGGEDIN); + this.setGaugeValue(gauges, MetricNames.USERS, calculateAllUsers(), MetricLabels.TOTAL); + + this.setGaugeValue(gauges, MetricNames.LANGUAGES, calculateLanguages(), null); + this.setGaugeValue(gauges, MetricNames.INSTALLATIONS, 1d, null); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.INSTALLATIONS, 1d, null); } finally { this.entityManager.enableTenantFilters(); } @@ -140,10 +176,54 @@ public class MetricsServiceImpl implements MetricsService { } private double calculateDoiedDmps(boolean forNexus) { - EntityDoiQuery entityDoiQuery = this.queryFactory.query(EntityDoiQuery.class).types(EntityType.DMP).isActive(IsActive.Active); + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).isActive(IsActive.Active); if (forNexus) - entityDoiQuery.after(_config.getNexusDate()); - return entityDoiQuery.count(); + dmpQuery.after(_config.getNexusDate()); + EntityDoiQuery entityDoiQuery = this.queryFactory.query(EntityDoiQuery.class).types(EntityType.DMP).isActive(IsActive.Active); + dmpQuery.entityDoiSubQuery(entityDoiQuery); + return dmpQuery.count(); + } + + private double calculateDraftDmpsWithGrant(boolean forNexus) { + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Draft).isActive(IsActive.Active); + if (forNexus) + dmpQuery.after(_config.getNexusDate()); + ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); + DmpReferenceQuery dmpReferenceQuery = this.queryFactory.query(DmpReferenceQuery.class).referenceSubQuery(referenceQuery).isActives(IsActive.Active); + dmpQuery.dmpReferenceSubQuery(dmpReferenceQuery); + return dmpQuery.count(); + } + + private double calculateFinalizedDmpsWithGrant(boolean forNexus) { + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).isActive(IsActive.Active); + if (forNexus) + dmpQuery.after(_config.getNexusDate()); + ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); + DmpReferenceQuery dmpReferenceQuery = this.queryFactory.query(DmpReferenceQuery.class).referenceSubQuery(referenceQuery).isActives(IsActive.Active); + dmpQuery.dmpReferenceSubQuery(dmpReferenceQuery); + return dmpQuery.count(); + } + + private double calculatePublishedDmpsWithGrant(boolean forNexus) { + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).accessTypes(DmpAccessType.Public).isActive(IsActive.Active); + if (forNexus) + dmpQuery.after(_config.getNexusDate()); + ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); + DmpReferenceQuery dmpReferenceQuery = this.queryFactory.query(DmpReferenceQuery.class).referenceSubQuery(referenceQuery).isActives(IsActive.Active); + dmpQuery.dmpReferenceSubQuery(dmpReferenceQuery); + return dmpQuery.count(); + } + + private double calculateDoiedDmpsWithGrant(boolean forNexus) { + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).isActive(IsActive.Active); + if (forNexus) + dmpQuery.after(_config.getNexusDate()); + ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); + DmpReferenceQuery dmpReferenceQuery = this.queryFactory.query(DmpReferenceQuery.class).referenceSubQuery(referenceQuery).isActives(IsActive.Active); + dmpQuery.dmpReferenceSubQuery(dmpReferenceQuery); + EntityDoiQuery entityDoiQuery = this.queryFactory.query(EntityDoiQuery.class).isActive(IsActive.Active); + dmpQuery.entityDoiSubQuery(entityDoiQuery); + return dmpQuery.count(); } private double calculateAllFunders(boolean forNexus) { @@ -223,6 +303,19 @@ public class MetricsServiceImpl implements MetricsService { return dmpDescriptionTemplateQuery.count(); } + private double calculateActiveUsers() { + return this.keycloakAdminRestApi.users().getUserSessionsCountByClientId(_keycloakConfig.getClientId()); + } + + private double calculateAllUsers() { + return this.keycloakAdminRestApi.users().getAllUsersCount(); + } + + private double calculateLanguages() { + LanguageQuery languageQuery = this.queryFactory.query(LanguageQuery.class).isActive(IsActive.Active); + return languageQuery.count(); + } + private void setGaugeValue(Map gauges, String name, Double amount, String label) { if (label != null) { gauges.get(name).labels(label).set(amount);