From d5e43e12bf8041154b513a216da5350ad75b293f Mon Sep 17 00:00:00 2001 From: Thomas Georgios Giannos Date: Fri, 12 Apr 2024 13:49:13 +0300 Subject: [PATCH] Adding back metrics for dmps, datasets, dataset templates and references for prometheus --- .../eudat/commons/metrics/MetricLabels.java | 21 ++ .../eu/eudat/commons/metrics/MetricNames.java | 26 +- .../java/eu/eudat/query/DescriptionQuery.java | 103 ++++--- .../eudat/query/DescriptionTemplateQuery.java | 13 + .../query/DmpDescriptionTemplateQuery.java | 82 ++++-- .../main/java/eu/eudat/query/DmpQuery.java | 120 ++++++--- .../java/eu/eudat/query/EntityDoiQuery.java | 49 ++-- .../java/eu/eudat/query/ReferenceQuery.java | 98 ++++--- .../service/metrics/MetricsServiceImpl.java | 255 ++++++++++++++---- .../metrics/UpdateMetricsTaskProperties.java | 99 +++++-- .../eudat/logic/managers/MetricsManager.java | 5 +- .../web/src/main/resources/config/metrics.yml | 8 +- 12 files changed, 643 insertions(+), 236 deletions(-) create mode 100644 dmp-backend/core/src/main/java/eu/eudat/commons/metrics/MetricLabels.java diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/metrics/MetricLabels.java b/dmp-backend/core/src/main/java/eu/eudat/commons/metrics/MetricLabels.java new file mode 100644 index 000000000..b8cabe8c6 --- /dev/null +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/metrics/MetricLabels.java @@ -0,0 +1,21 @@ +package eu.eudat.commons.metrics; + +public class MetricLabels { + + public static final String DRAFT = "draft"; + + public static final String FINALIZED = "finalized"; + + public static final String PUBLISHED = "published"; + + public static final String DOIED = "doied"; + + public static final String ACTIVE = "active"; + + public static final String USED = "used"; + + public static final String LOGGEDIN = "loggedin"; + + public static final String TOTAL = "total"; + +} diff --git a/dmp-backend/core/src/main/java/eu/eudat/commons/metrics/MetricNames.java b/dmp-backend/core/src/main/java/eu/eudat/commons/metrics/MetricNames.java index cb4c80f3b..b600f77f5 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/commons/metrics/MetricNames.java +++ b/dmp-backend/core/src/main/java/eu/eudat/commons/metrics/MetricNames.java @@ -1,24 +1,28 @@ package eu.eudat.commons.metrics; public class MetricNames { + public static final String DATASET_TEMPLATE = "argos_dmp_templates"; + public static final String INSTALLATIONS = "installations"; + public static final String USERS = "argos_users"; + public static final String DMP = "argos_managed_dmps"; + public static final String DATASET = "argos_managed_dataset_descriptions"; - public static final String RESEARCHER = "argos_researchers"; - public static final String PROJECT = "argos_projects"; + + public static final String RESEARCHERS = "argos_researchers"; + + public static final String PROJECTS = "argos_projects"; + public static final String FUNDERS = "argos_funders"; + public static final String GRANTS = "argos_grants"; + public static final String LANGUAGES = "argos_languages"; + public static final String DMP_WITH_GRANT = "argos_managed_dmps_with_grantid"; - public static final String DRAFT = "draft"; - public static final String FINALIZED = "finalized"; - public static final String PUBLISHED = "published"; - public static final String DOIED = "doied"; - public static final String ACTIVE = "active"; - public static final String USED = "used"; - public static final String LOGGEDIN = "loggedin"; - public static final String TOTAL = "total"; - public static final String NEXUS = "nexus_"; + + public static final String NEXUS_PREFIX = "nexus_"; } diff --git a/dmp-backend/core/src/main/java/eu/eudat/query/DescriptionQuery.java b/dmp-backend/core/src/main/java/eu/eudat/query/DescriptionQuery.java index 90a848baf..a2267de77 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/query/DescriptionQuery.java +++ b/dmp-backend/core/src/main/java/eu/eudat/query/DescriptionQuery.java @@ -3,17 +3,13 @@ package eu.eudat.query; import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.authorization.Permission; import eu.eudat.commons.enums.DescriptionStatus; -import eu.eudat.commons.enums.DmpAccessType; import eu.eudat.commons.enums.IsActive; import eu.eudat.commons.scope.user.UserScope; import eu.eudat.data.DescriptionEntity; import eu.eudat.data.DmpDescriptionTemplateEntity; import eu.eudat.data.DmpEntity; import eu.eudat.model.Description; -import eu.eudat.model.Dmp; -import eu.eudat.model.DmpDescriptionTemplate; import eu.eudat.model.PublicDescription; -import eu.eudat.query.utils.BuildSubQueryInput; import eu.eudat.query.utils.QueryUtilsService; import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.tools.data.query.FieldResolver; @@ -22,7 +18,6 @@ import gr.cite.tools.data.query.QueryContext; import jakarta.persistence.Tuple; import jakarta.persistence.criteria.CriteriaBuilder; import jakarta.persistence.criteria.Predicate; -import jakarta.persistence.criteria.Subquery; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -33,29 +28,43 @@ import java.util.*; @Component @Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE) public class DescriptionQuery extends QueryBase { + private String like; + private Collection ids; + private Collection createdByIds; - + private DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery; + private DmpQuery dmpQuery; + private Instant createdAfter; + private Instant createdBefore; + private Instant finalizedAfter; + private Instant finalizedBefore; + private Collection excludedIds; + private Collection isActives; private Collection statuses; + private Collection dmpIds; private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); private final UserScope userScope; + private final AuthorizationService authService; + private final QueryUtilsService queryUtilsService; private Collection dmpDescriptionTemplateIds; + public DescriptionQuery(UserScope userScope, AuthorizationService authService, QueryUtilsService queryUtilsService) { this.userScope = userScope; this.authService = authService; @@ -141,7 +150,7 @@ public class DescriptionQuery extends QueryBase { this.dmpQuery = subQuery; return this; } - + public DescriptionQuery excludedIds(Collection values) { this.excludedIds = values; return this; @@ -218,15 +227,19 @@ public class DescriptionQuery extends QueryBase { @Override protected Predicate applyAuthZ(QueryContext queryContext) { - if (this.authorize.contains(AuthorizationFlags.None)) return null; - if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseDescription)) return null; + if (this.authorize.contains(AuthorizationFlags.None)) + return null; + if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseDescription)) + return null; UUID userId; boolean usePublic = this.authorize.contains(AuthorizationFlags.Public); - if (this.authorize.contains(AuthorizationFlags.DmpAssociated)) userId = this.userScope.getUserIdSafe(); - else userId = null; + if (this.authorize.contains(AuthorizationFlags.DmpAssociated)) + userId = this.userScope.getUserIdSafe(); + else + userId = null; List predicates = new ArrayList<>(); - if (userId != null || usePublic ) { + if (userId != null || usePublic) { predicates.add(queryContext.CriteriaBuilder.in(queryContext.Root.get(DescriptionEntity._id)).value(queryUtilsService.buildDescriptionAuthZSubQuery(queryContext.Query, queryContext.CriteriaBuilder, userId, usePublic))); } if (!predicates.isEmpty()) { @@ -236,7 +249,7 @@ public class DescriptionQuery extends QueryBase { return queryContext.CriteriaBuilder.or(); //Creates a false query } } - + @Override protected Predicate applyFilters(QueryContext queryContext) { List predicates = new ArrayList<>(); @@ -308,6 +321,7 @@ public class DescriptionQuery extends QueryBase { inClause.value(item); predicates.add(inClause); } + if (!predicates.isEmpty()) { Predicate[] predicatesArray = predicates.toArray(new Predicate[0]); return queryContext.CriteriaBuilder.and(predicatesArray); @@ -318,27 +332,48 @@ public class DescriptionQuery extends QueryBase { @Override protected String fieldNameOf(FieldResolver item) { - if (item.match(Description._id) || item.match(PublicDescription._id)) return DescriptionEntity._id; - else if (item.match(Description._label) || item.match(PublicDescription._label)) return DescriptionEntity._label; - else if (item.prefix(Description._properties)) return DescriptionEntity._properties; - else if (item.match(Description._status) || item.match(PublicDescription._status)) return DescriptionEntity._status; - else if (item.match(Description._description) || item.match(PublicDescription._description)) return DescriptionEntity._description; - else if (item.match(Description._createdBy)) return DescriptionEntity._createdById; - else if (item.prefix(Description._createdBy)) return DescriptionEntity._createdById; - else if (item.match(Description._createdAt) || item.match(PublicDescription._createdAt)) return DescriptionEntity._createdAt; - else if (item.match(Description._updatedAt) || item.match(PublicDescription._updatedAt)) return DescriptionEntity._updatedAt; - else if (item.match(Description._isActive)) return DescriptionEntity._isActive; - else if (item.match(Description._hash)) return DescriptionEntity._updatedAt; - else if (item.match(Description._finalizedAt) || item.match(PublicDescription._finalizedAt)) return DescriptionEntity._finalizedAt; - else if (item.prefix(Description._dmpDescriptionTemplate) || item.prefix(PublicDescription._dmpDescriptionTemplate)) return DescriptionEntity._dmpDescriptionTemplateId; - else if (item.match(Description._dmpDescriptionTemplate) || item.match(PublicDescription._dmpDescriptionTemplate)) return DescriptionEntity._dmpDescriptionTemplateId; - else if (item.prefix(Description._descriptionTemplate) || item.prefix(PublicDescription._descriptionTemplate)) return DescriptionEntity._descriptionTemplateId; - else if (item.match(Description._descriptionTemplate) || item.match(PublicDescription._descriptionTemplate)) return DescriptionEntity._descriptionTemplateId; - else if (item.prefix(Description._dmp)) return DescriptionEntity._dmpId; - else if (item.match(Description._dmp)) return DescriptionEntity._dmpId; - else if (item.match(Description._belongsToCurrentTenant)) return DescriptionEntity._tenantId; - else if (item.match(DescriptionEntity._tenantId)) return DescriptionEntity._tenantId; - else return null; + if (item.match(Description._id) || item.match(PublicDescription._id)) + return DescriptionEntity._id; + else if (item.match(Description._label) || item.match(PublicDescription._label)) + return DescriptionEntity._label; + else if (item.prefix(Description._properties)) + return DescriptionEntity._properties; + else if (item.match(Description._status) || item.match(PublicDescription._status)) + return DescriptionEntity._status; + else if (item.match(Description._description) || item.match(PublicDescription._description)) + return DescriptionEntity._description; + else if (item.match(Description._createdBy)) + return DescriptionEntity._createdById; + else if (item.prefix(Description._createdBy)) + return DescriptionEntity._createdById; + else if (item.match(Description._createdAt) || item.match(PublicDescription._createdAt)) + return DescriptionEntity._createdAt; + else if (item.match(Description._updatedAt) || item.match(PublicDescription._updatedAt)) + return DescriptionEntity._updatedAt; + else if (item.match(Description._isActive)) + return DescriptionEntity._isActive; + else if (item.match(Description._hash)) + return DescriptionEntity._updatedAt; + else if (item.match(Description._finalizedAt) || item.match(PublicDescription._finalizedAt)) + return DescriptionEntity._finalizedAt; + else if (item.prefix(Description._dmpDescriptionTemplate) || item.prefix(PublicDescription._dmpDescriptionTemplate)) + return DescriptionEntity._dmpDescriptionTemplateId; + else if (item.match(Description._dmpDescriptionTemplate) || item.match(PublicDescription._dmpDescriptionTemplate)) + return DescriptionEntity._dmpDescriptionTemplateId; + else if (item.prefix(Description._descriptionTemplate) || item.prefix(PublicDescription._descriptionTemplate)) + return DescriptionEntity._descriptionTemplateId; + else if (item.match(Description._descriptionTemplate) || item.match(PublicDescription._descriptionTemplate)) + return DescriptionEntity._descriptionTemplateId; + else if (item.prefix(Description._dmp)) + return DescriptionEntity._dmpId; + else if (item.match(Description._dmp)) + return DescriptionEntity._dmpId; + else if (item.match(Description._belongsToCurrentTenant)) + return DescriptionEntity._tenantId; + else if (item.match(DescriptionEntity._tenantId)) + return DescriptionEntity._tenantId; + else + return null; } @Override diff --git a/dmp-backend/core/src/main/java/eu/eudat/query/DescriptionTemplateQuery.java b/dmp-backend/core/src/main/java/eu/eudat/query/DescriptionTemplateQuery.java index 0c8a2236a..c3f7e4298 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/query/DescriptionTemplateQuery.java +++ b/dmp-backend/core/src/main/java/eu/eudat/query/DescriptionTemplateQuery.java @@ -8,6 +8,7 @@ import eu.eudat.commons.enums.IsActive; import eu.eudat.commons.scope.user.UserScope; import eu.eudat.data.DescriptionTemplateEntity; import eu.eudat.data.DmpDescriptionTemplateEntity; +import eu.eudat.data.DmpEntity; import eu.eudat.model.DescriptionTemplate; import eu.eudat.query.utils.BuildSubQueryInput; import eu.eudat.query.utils.QueryUtilsService; @@ -50,6 +51,8 @@ public class DescriptionTemplateQuery extends QueryBase typeIds; + private Instant after; + private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); public DescriptionTemplateQuery like(String value) { @@ -192,6 +195,11 @@ public class DescriptionTemplateQuery extends QueryBase values) { this.authorize = values; return this; @@ -314,6 +322,11 @@ public class DescriptionTemplateQuery extends QueryBase excludedIds; - private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); + private Instant after; + private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); public DmpDescriptionTemplateQuery ids(UUID value) { this.ids = List.of(value); @@ -105,8 +101,8 @@ public class DmpDescriptionTemplateQuery extends QueryBase values) { this.sectionIds = values; return this; - } - + } + public DmpDescriptionTemplateQuery descriptionTemplateGroupIds(UUID value) { this.descriptionTemplateGroupIds = List.of(value); return this; @@ -137,13 +133,20 @@ public class DmpDescriptionTemplateQuery extends QueryBase values) { this.authorize = values; return this; } private final UserScope userScope; + private final AuthorizationService authService; + private final QueryUtilsService queryUtilsService; public DmpDescriptionTemplateQuery( @@ -165,14 +168,17 @@ public class DmpDescriptionTemplateQuery extends QueryBase Predicate applyAuthZ(QueryContext queryContext) { - if (this.authorize.contains(AuthorizationFlags.None)) return null; - if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseDmpDescriptionTemplate)) return null; + if (this.authorize.contains(AuthorizationFlags.None)) + return null; + if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseDmpDescriptionTemplate)) + return null; UUID userId = null; boolean usePublic = this.authorize.contains(AuthorizationFlags.Public); - if (this.authorize.contains(AuthorizationFlags.DmpAssociated)) userId = this.userScope.getUserIdSafe(); + if (this.authorize.contains(AuthorizationFlags.DmpAssociated)) + userId = this.userScope.getUserIdSafe(); List predicates = new ArrayList<>(); - if (userId != null || usePublic ) { + if (userId != null || usePublic) { predicates.add(queryContext.CriteriaBuilder.in(queryContext.Root.get(DmpDescriptionTemplateEntity._dmpId)).value(queryUtilsService.buildDmpAuthZSubQuery(queryContext.Query, queryContext.CriteriaBuilder, userId, usePublic))); } if (!predicates.isEmpty()) { @@ -182,7 +188,7 @@ public class DmpDescriptionTemplateQuery extends QueryBase Predicate applyFilters(QueryContext queryContext) { List predicates = new ArrayList<>(); @@ -222,6 +228,11 @@ public class DmpDescriptionTemplateQuery extends QueryBase { private String like; private Collection ids; + private Collection creatorIds; private Collection excludedIds; @@ -38,22 +42,29 @@ public class DmpQuery extends QueryBase { private Collection isActives; private Collection statuses; + private Collection versionStatuses; + private Collection accessTypes; private Collection versions; private Collection groupIds; + + private Instant after; + private DmpUserQuery dmpUserQuery; private DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery; - + private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); private final UserScope userScope; + private final AuthorizationService authService; + private final QueryUtilsService queryUtilsService; - + public DmpQuery(UserScope userScope, AuthorizationService authService, QueryUtilsService queryUtilsService) { this.userScope = userScope; this.authService = authService; @@ -200,6 +211,11 @@ public class DmpQuery extends QueryBase { return this; } + public DmpQuery after(Instant value) { + this.after = value; + return this; + } + public DmpQuery dmpDescriptionTemplateSubQuery(DmpDescriptionTemplateQuery subQuery) { this.dmpDescriptionTemplateQuery = subQuery; return this; @@ -217,7 +233,7 @@ public class DmpQuery extends QueryBase { @Override protected Boolean isFalseQuery() { - return this.isEmpty(this.ids) || this.isEmpty(this.creatorIds) ||this.isEmpty(this.isActives) || this.isEmpty(this.versionStatuses) || this.isEmpty(this.excludedIds) || this.isEmpty(this.accessTypes)|| this.isEmpty(this.statuses)|| this.isFalseQuery(this.dmpDescriptionTemplateQuery)|| this.isFalseQuery(this.dmpUserQuery); + return this.isEmpty(this.ids) || this.isEmpty(this.creatorIds) || this.isEmpty(this.isActives) || this.isEmpty(this.versionStatuses) || this.isEmpty(this.excludedIds) || this.isEmpty(this.accessTypes) || this.isEmpty(this.statuses) || this.isFalseQuery(this.dmpDescriptionTemplateQuery) || this.isFalseQuery(this.dmpUserQuery); } @Override @@ -227,19 +243,22 @@ public class DmpQuery extends QueryBase { @Override protected Predicate applyAuthZ(QueryContext queryContext) { - if (this.authorize.contains(AuthorizationFlags.None)) return null; - if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseDmp)) return null; + if (this.authorize.contains(AuthorizationFlags.None)) + return null; + if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseDmp)) + return null; UUID userId = null; boolean usePublic = this.authorize.contains(AuthorizationFlags.Public); - if (this.authorize.contains(AuthorizationFlags.DmpAssociated)) userId = this.userScope.getUserIdSafe(); - + if (this.authorize.contains(AuthorizationFlags.DmpAssociated)) + userId = this.userScope.getUserIdSafe(); + List predicates = new ArrayList<>(); - if (userId != null || usePublic ) { + if (userId != null || usePublic) { predicates.add(queryContext.CriteriaBuilder.or( usePublic ? queryContext.CriteriaBuilder.and( - queryContext.CriteriaBuilder.equal(queryContext.Root.get(DmpEntity._status), DmpStatus.Finalized), - queryContext.CriteriaBuilder.equal(queryContext.Root.get(DmpEntity._accessType), DmpAccessType.Public) - ) + queryContext.CriteriaBuilder.equal(queryContext.Root.get(DmpEntity._status), DmpStatus.Finalized), + queryContext.CriteriaBuilder.equal(queryContext.Root.get(DmpEntity._accessType), DmpAccessType.Public) + ) : queryContext.CriteriaBuilder.or(), //Creates a false query userId != null ? queryContext.CriteriaBuilder.in(queryContext.Root.get(DmpEntity._id)).value(this.queryUtilsService.buildDmpUserAuthZSubQuery(queryContext.Query, queryContext.CriteriaBuilder, userId)) : queryContext.CriteriaBuilder.or() //Creates a false query )); @@ -251,7 +270,7 @@ public class DmpQuery extends QueryBase { return queryContext.CriteriaBuilder.or(); //Creates a false query } } - + @Override protected Predicate applyFilters(QueryContext queryContext) { List predicates = new ArrayList<>(); @@ -312,6 +331,10 @@ public class DmpQuery extends QueryBase { inClause.value(item); predicates.add(inClause); } + if (this.after != null) { + Predicate afterClause = queryContext.CriteriaBuilder.greaterThanOrEqualTo(queryContext.Root.get(DmpEntity._createdAt), this.after); + predicates.add(afterClause); + } if (this.dmpUserQuery != null) { QueryContext subQuery = this.applySubQuery(this.dmpUserQuery, queryContext, UUID.class, root -> root.get(DmpUserEntity._dmpId)); @@ -332,29 +355,52 @@ public class DmpQuery extends QueryBase { @Override protected String fieldNameOf(FieldResolver item) { - if (item.match(Dmp._id) || item.match(PublicDmp._id)) return DmpEntity._id; - else if (item.match(Dmp._label) || item.match(PublicDmp._label)) return DmpEntity._label; - else if (item.match(Dmp._version) || item.match(PublicDmp._version)) return DmpEntity._version; - else if (item.match(Dmp._status)) return DmpEntity._status; - else if (item.match(Dmp._properties)) return DmpEntity._properties; - else if (item.prefix(Dmp._properties)) return DmpEntity._properties; - else if (item.match(Dmp._groupId)) return DmpEntity._groupId; - else if (item.match(Dmp._description) || item.match(PublicDmp._description)) return DmpEntity._description; - else if (item.match(Dmp._createdAt)) return DmpEntity._createdAt; - else if (item.match(Dmp._updatedAt)) return DmpEntity._updatedAt; - else if (item.match(Dmp._hash)) return DmpEntity._updatedAt; - else if (item.match(Dmp._isActive) ) return DmpEntity._isActive; - else if (item.match(Dmp._finalizedAt) || item.match(PublicDmp._finalizedAt)) return DmpEntity._finalizedAt; - else if (item.match(Dmp._accessType)) return DmpEntity._accessType; - else if (item.match(Dmp._creator)) return DmpEntity._creatorId; - else if (item.prefix(Dmp._creator)) return DmpEntity._blueprintId; - else if (item.match(Dmp._blueprint)) return DmpEntity._blueprintId; - else if (item.prefix(Dmp._blueprint)) return DmpEntity._blueprintId; - else if (item.match(Dmp._language)) return DmpEntity._language; - else if (item.match(Dmp._publicAfter)) return DmpEntity._publicAfter; - else if (item.match(Dmp._versionStatus)) return DmpEntity._versionStatus; - else if (item.match(Dmp._belongsToCurrentTenant)) return DmpEntity._tenantId; - else return null; + if (item.match(Dmp._id) || item.match(PublicDmp._id)) + return DmpEntity._id; + else if (item.match(Dmp._label) || item.match(PublicDmp._label)) + return DmpEntity._label; + else if (item.match(Dmp._version) || item.match(PublicDmp._version)) + return DmpEntity._version; + else if (item.match(Dmp._status)) + return DmpEntity._status; + else if (item.match(Dmp._properties)) + return DmpEntity._properties; + else if (item.prefix(Dmp._properties)) + return DmpEntity._properties; + else if (item.match(Dmp._groupId)) + return DmpEntity._groupId; + else if (item.match(Dmp._description) || item.match(PublicDmp._description)) + return DmpEntity._description; + else if (item.match(Dmp._createdAt)) + return DmpEntity._createdAt; + else if (item.match(Dmp._updatedAt)) + return DmpEntity._updatedAt; + else if (item.match(Dmp._hash)) + return DmpEntity._updatedAt; + else if (item.match(Dmp._isActive)) + return DmpEntity._isActive; + else if (item.match(Dmp._finalizedAt) || item.match(PublicDmp._finalizedAt)) + return DmpEntity._finalizedAt; + else if (item.match(Dmp._accessType)) + return DmpEntity._accessType; + else if (item.match(Dmp._creator)) + return DmpEntity._creatorId; + else if (item.prefix(Dmp._creator)) + return DmpEntity._blueprintId; + else if (item.match(Dmp._blueprint)) + return DmpEntity._blueprintId; + else if (item.prefix(Dmp._blueprint)) + return DmpEntity._blueprintId; + else if (item.match(Dmp._language)) + return DmpEntity._language; + else if (item.match(Dmp._publicAfter)) + return DmpEntity._publicAfter; + else if (item.match(Dmp._versionStatus)) + return DmpEntity._versionStatus; + else if (item.match(Dmp._belongsToCurrentTenant)) + return DmpEntity._tenantId; + else + return null; } @Override diff --git a/dmp-backend/core/src/main/java/eu/eudat/query/EntityDoiQuery.java b/dmp-backend/core/src/main/java/eu/eudat/query/EntityDoiQuery.java index a9dc30cd6..9bc68da5b 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/query/EntityDoiQuery.java +++ b/dmp-backend/core/src/main/java/eu/eudat/query/EntityDoiQuery.java @@ -5,10 +5,9 @@ import eu.eudat.authorization.Permission; import eu.eudat.commons.enums.EntityType; import eu.eudat.commons.enums.IsActive; import eu.eudat.commons.scope.user.UserScope; -import eu.eudat.data.*; -import eu.eudat.model.Dmp; +import eu.eudat.data.DmpEntity; +import eu.eudat.data.EntityDoiEntity; import eu.eudat.model.EntityDoi; -import eu.eudat.query.utils.BuildSubQueryInput; import eu.eudat.query.utils.QueryUtilsService; import gr.cite.commons.web.authz.service.AuthorizationService; import gr.cite.tools.data.query.FieldResolver; @@ -17,7 +16,6 @@ import gr.cite.tools.data.query.QueryContext; import jakarta.persistence.Tuple; import jakarta.persistence.criteria.CriteriaBuilder; import jakarta.persistence.criteria.Predicate; -import jakarta.persistence.criteria.Subquery; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; import org.springframework.stereotype.Component; @@ -36,12 +34,15 @@ public class EntityDoiQuery extends QueryBase { private Collection types; private Collection excludedIds; + private Collection repositoryIds; private Collection dois; private Collection entityIds; + private Instant after; + private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); public EntityDoiQuery ids(UUID value) { @@ -149,6 +150,11 @@ public class EntityDoiQuery extends QueryBase { return this; } + public EntityDoiQuery after(Instant value) { + this.after = value; + return this; + } + public EntityDoiQuery authorize(EnumSet values) { this.authorize = values; return this; @@ -160,13 +166,11 @@ public class EntityDoiQuery extends QueryBase { private final QueryUtilsService queryUtilsService; - - public EntityDoiQuery( - UserScope userScope, AuthorizationService authService, QueryUtilsService queryUtilsService) { - this.userScope = userScope; - this.authService = authService; - this.queryUtilsService = queryUtilsService; + UserScope userScope, AuthorizationService authService, QueryUtilsService queryUtilsService) { + this.userScope = userScope; + this.authService = authService; + this.queryUtilsService = queryUtilsService; } @Override @@ -176,16 +180,20 @@ public class EntityDoiQuery extends QueryBase { @Override protected Boolean isFalseQuery() { - return this.isEmpty(this.ids) || this.isEmpty(this.isActives)|| this.isEmpty(this.repositoryIds) || this.isEmpty(this.excludedIds) || this.isEmpty(this.types); + return this.isEmpty(this.ids) || this.isEmpty(this.isActives) || this.isEmpty(this.repositoryIds) || this.isEmpty(this.excludedIds) || this.isEmpty(this.types); } @Override protected Predicate applyAuthZ(QueryContext queryContext) { - if (this.authorize.contains(AuthorizationFlags.None)) return null; - if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseUser)) return null; + if (this.authorize.contains(AuthorizationFlags.None)) + return null; + if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseUser)) + return null; UUID userId; - if (this.authorize.contains(AuthorizationFlags.Owner)) userId = this.userScope.getUserIdSafe(); - else userId = null; + if (this.authorize.contains(AuthorizationFlags.Owner)) + userId = this.userScope.getUserIdSafe(); + else + userId = null; List predicates = new ArrayList<>(); boolean usePublic = this.authorize.contains(AuthorizationFlags.Public); @@ -240,13 +248,17 @@ public class EntityDoiQuery extends QueryBase { inClause.value(item); predicates.add(inClause); } - if (this.entityIds != null) { CriteriaBuilder.In inClause = queryContext.CriteriaBuilder.in(queryContext.Root.get(EntityDoiEntity._entityId)); - for (UUID item: this.entityIds) + for (UUID item : this.entityIds) inClause.value(item); predicates.add(inClause); } + if (this.after != null) { + Predicate afterClause = queryContext.CriteriaBuilder.greaterThanOrEqualTo(queryContext.Root.get(DmpEntity._createdAt), this.after); + predicates.add(afterClause); + } + if (!predicates.isEmpty()) { Predicate[] predicatesArray = predicates.toArray(new Predicate[0]); return queryContext.CriteriaBuilder.and(predicatesArray); @@ -286,7 +298,8 @@ public class EntityDoiQuery extends QueryBase { return EntityDoiEntity._createdAt; else if (item.match(EntityDoi._updatedAt)) return EntityDoiEntity._updatedAt; - else if (item.match(EntityDoi._hash)) return EntityDoiEntity._updatedAt; + else if (item.match(EntityDoi._hash)) + return EntityDoiEntity._updatedAt; else if (item.match(EntityDoi._isActive)) return EntityDoiEntity._isActive; else if (item.match(EntityDoi._belongsToCurrentTenant)) diff --git a/dmp-backend/core/src/main/java/eu/eudat/query/ReferenceQuery.java b/dmp-backend/core/src/main/java/eu/eudat/query/ReferenceQuery.java index 9b7d0235f..a24fda1ae 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/query/ReferenceQuery.java +++ b/dmp-backend/core/src/main/java/eu/eudat/query/ReferenceQuery.java @@ -2,14 +2,11 @@ package eu.eudat.query; import eu.eudat.authorization.AuthorizationFlags; import eu.eudat.authorization.Permission; -import eu.eudat.commons.enums.DmpAccessType; -import eu.eudat.commons.enums.DmpStatus; import eu.eudat.commons.enums.IsActive; import eu.eudat.commons.enums.ReferenceSourceType; import eu.eudat.commons.scope.user.UserScope; import eu.eudat.data.DmpEntity; import eu.eudat.data.DmpReferenceEntity; -import eu.eudat.data.DmpUserEntity; import eu.eudat.data.ReferenceEntity; import eu.eudat.model.PublicReference; import eu.eudat.model.Reference; @@ -44,12 +41,14 @@ public class ReferenceQuery extends QueryBase { private Collection typeIds; private Collection references; + private Collection sources; private Collection excludedIds; - private DmpReferenceQuery dmpReferenceQuery; + private Instant after; + private DmpReferenceQuery dmpReferenceQuery; private EnumSet authorize = EnumSet.of(AuthorizationFlags.None); @@ -163,6 +162,11 @@ public class ReferenceQuery extends QueryBase { return this; } + public ReferenceQuery after(Instant value) { + this.after = value; + return this; + } + public ReferenceQuery dmpReferenceSubQuery(DmpReferenceQuery value) { this.dmpReferenceQuery = value; return this; @@ -172,16 +176,18 @@ public class ReferenceQuery extends QueryBase { this.authorize = values; return this; } + private final UserScope userScope; private final AuthorizationService authService; + private final QueryUtilsService queryUtilsService; public ReferenceQuery( - UserScope userScope, AuthorizationService authService, QueryUtilsService queryUtilsService) { - this.userScope = userScope; - this.authService = authService; - this.queryUtilsService = queryUtilsService; + UserScope userScope, AuthorizationService authService, QueryUtilsService queryUtilsService) { + this.userScope = userScope; + this.authService = authService; + this.queryUtilsService = queryUtilsService; } @Override @@ -191,23 +197,27 @@ public class ReferenceQuery extends QueryBase { @Override protected Boolean isFalseQuery() { - return this.isEmpty(this.ids) || this.isEmpty(this.isActives) || this.isEmpty(this.sources) ||this.isEmpty(this.excludedIds) || this.isEmpty(this.typeIds) || this.isEmpty(this.referenceSourceTypes)|| this.isFalseQuery(this.dmpReferenceQuery); + return this.isEmpty(this.ids) || this.isEmpty(this.isActives) || this.isEmpty(this.sources) || this.isEmpty(this.excludedIds) || this.isEmpty(this.typeIds) || this.isEmpty(this.referenceSourceTypes) || this.isFalseQuery(this.dmpReferenceQuery); } @Override protected Predicate applyAuthZ(QueryContext queryContext) { - if (this.authorize.contains(AuthorizationFlags.None)) return null; - if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseReference)) return null; + if (this.authorize.contains(AuthorizationFlags.None)) + return null; + if (this.authorize.contains(AuthorizationFlags.Permission) && this.authService.authorize(Permission.BrowseReference)) + return null; UUID userId; boolean usePublic = this.authorize.contains(AuthorizationFlags.Public); - if (this.authorize.contains(AuthorizationFlags.DmpAssociated)) userId = this.userScope.getUserIdSafe(); - else userId = null; + if (this.authorize.contains(AuthorizationFlags.DmpAssociated)) + userId = this.userScope.getUserIdSafe(); + else + userId = null; List predicates = new ArrayList<>(); - if (userId != null || usePublic ) { + if (userId != null || usePublic) { predicates.add(queryContext.CriteriaBuilder.or( queryContext.CriteriaBuilder.isNull(queryContext.Root.get(ReferenceEntity._createdById)), - userId != null ? queryContext.CriteriaBuilder.equal(queryContext.Root.get(ReferenceEntity._createdById), userId) : queryContext.CriteriaBuilder.or(), //Creates a false query + userId != null ? queryContext.CriteriaBuilder.equal(queryContext.Root.get(ReferenceEntity._createdById), userId) : queryContext.CriteriaBuilder.or(), //Creates a false query queryContext.CriteriaBuilder.in(queryContext.Root.get(ReferenceEntity._id)).value(queryUtilsService.buildSubQuery(new BuildSubQueryInput<>(new BuildSubQueryInput.Builder<>(DmpReferenceEntity.class, UUID.class) .query(queryContext.Query) .criteriaBuilder(queryContext.CriteriaBuilder) @@ -225,7 +235,7 @@ public class ReferenceQuery extends QueryBase { return queryContext.CriteriaBuilder.or(); //Creates a false query } } - + @Override protected Predicate applyFilters(QueryContext queryContext) { List predicates = new ArrayList<>(); @@ -277,6 +287,11 @@ public class ReferenceQuery extends QueryBase { notInClause.value(item); predicates.add(notInClause.not()); } + if (this.after != null) { + Predicate afterClause = queryContext.CriteriaBuilder.greaterThanOrEqualTo(queryContext.Root.get(DmpEntity._createdAt), this.after); + predicates.add(afterClause); + } + if (this.dmpReferenceQuery != null) { QueryContext subQuery = this.applySubQuery(this.dmpReferenceQuery, queryContext, UUID.class, root -> root.get(DmpReferenceEntity._referenceId)); predicates.add(queryContext.CriteriaBuilder.in(queryContext.Root.get(DmpReferenceEntity._id)).value(subQuery.Query)); @@ -311,23 +326,40 @@ public class ReferenceQuery extends QueryBase { @Override protected String fieldNameOf(FieldResolver item) { - if (item.match(Reference._id) || item.match(PublicReference._id)) return ReferenceEntity._id; - else if (item.match(Reference._label) || item.match(PublicReference._label)) return ReferenceEntity._label; - else if (item.match(Reference._description) || item.match(PublicReference._description)) return ReferenceEntity._description; - else if (item.match(Reference._createdAt)) return ReferenceEntity._createdAt; - else if (item.match(Reference._updatedAt)) return ReferenceEntity._updatedAt; - else if (item.match(Reference._hash)) return ReferenceEntity._updatedAt; - else if (item.match(Reference._isActive)) return ReferenceEntity._isActive; - else if (item.prefix(Reference._definition)) return ReferenceEntity._definition; - else if (item.match(Reference._abbreviation)) return ReferenceEntity._abbreviation; - else if (item.match(Reference._reference) || item.match(PublicReference._reference)) return ReferenceEntity._reference; - else if (item.match(Reference._source)) return ReferenceEntity._source; - else if (item.match(Reference._sourceType)) return ReferenceEntity._sourceType; - else if (item.match(Reference._type) || item.match(PublicReference._type)) return ReferenceEntity._typeId; - else if (item.prefix(Reference._type) || item.prefix(PublicReference._type)) return ReferenceEntity._typeId; - else if (item.prefix(Reference._createdBy)) return ReferenceEntity._createdById; - else if (item.prefix(Reference._belongsToCurrentTenant)) return ReferenceEntity._tenantId; - else return null; + if (item.match(Reference._id) || item.match(PublicReference._id)) + return ReferenceEntity._id; + else if (item.match(Reference._label) || item.match(PublicReference._label)) + return ReferenceEntity._label; + else if (item.match(Reference._description) || item.match(PublicReference._description)) + return ReferenceEntity._description; + else if (item.match(Reference._createdAt)) + return ReferenceEntity._createdAt; + else if (item.match(Reference._updatedAt)) + return ReferenceEntity._updatedAt; + else if (item.match(Reference._hash)) + return ReferenceEntity._updatedAt; + else if (item.match(Reference._isActive)) + return ReferenceEntity._isActive; + else if (item.prefix(Reference._definition)) + return ReferenceEntity._definition; + else if (item.match(Reference._abbreviation)) + return ReferenceEntity._abbreviation; + else if (item.match(Reference._reference) || item.match(PublicReference._reference)) + return ReferenceEntity._reference; + else if (item.match(Reference._source)) + return ReferenceEntity._source; + else if (item.match(Reference._sourceType)) + return ReferenceEntity._sourceType; + else if (item.match(Reference._type) || item.match(PublicReference._type)) + return ReferenceEntity._typeId; + else if (item.prefix(Reference._type) || item.prefix(PublicReference._type)) + return ReferenceEntity._typeId; + else if (item.prefix(Reference._createdBy)) + return ReferenceEntity._createdById; + else if (item.prefix(Reference._belongsToCurrentTenant)) + return ReferenceEntity._tenantId; + else + return null; } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/metrics/MetricsServiceImpl.java b/dmp-backend/core/src/main/java/eu/eudat/service/metrics/MetricsServiceImpl.java index eb8133590..25e525c5c 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/metrics/MetricsServiceImpl.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/metrics/MetricsServiceImpl.java @@ -1,10 +1,10 @@ package eu.eudat.service.metrics; -import eu.eudat.commons.enums.DmpStatus; -import eu.eudat.commons.enums.IsActive; +import eu.eudat.commons.enums.*; +import eu.eudat.commons.metrics.MetricLabels; import eu.eudat.commons.metrics.MetricNames; import eu.eudat.data.TenantEntityManager; -import eu.eudat.query.DmpQuery; +import eu.eudat.query.*; import gr.cite.tools.data.query.QueryFactory; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.Gauge; @@ -17,76 +17,217 @@ import java.util.stream.Stream; @Service public class MetricsServiceImpl implements MetricsService { - private final PrometheusMeterRegistry registry; - private final QueryFactory queryFactory; - private final TenantEntityManager entityManager; - public MetricsServiceImpl(PrometheusMeterRegistry registry, QueryFactory queryFactory, TenantEntityManager entityManager) { - this.registry = registry; - this.queryFactory = queryFactory; - this.entityManager = entityManager; - } + private final PrometheusMeterRegistry registry; - @Override - public void calculate(Map gauges) throws InvalidApplicationException { - try { - this.entityManager.disableTenantFilters(); + private final QueryFactory queryFactory; - this.setGauseValue(gauges, MetricNames.DMP, calculateFinalizedDmps(), MetricNames.FINALIZED); - }finally { - this.entityManager.enableTenantFilters(); - } - } + private final TenantEntityManager entityManager; - @Override - public Map gaugesBuild() { - registry.clear(); + private final UpdateMetricsTaskProperties _config; - return Stream.of( new Object[][]{ - {MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())}, - {MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, + public MetricsServiceImpl(PrometheusMeterRegistry registry, QueryFactory queryFactory, TenantEntityManager entityManager, UpdateMetricsTaskProperties config) { + this.registry = registry; + this.queryFactory = queryFactory; + this.entityManager = entityManager; + this._config = config; + } - {MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())}, - {MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())}, - {MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())}, - {MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())}, + @Override + public void calculate(Map gauges) throws InvalidApplicationException { + try { + this.entityManager.disableTenantFilters(); - {MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())}, - {MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())}, - {MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())}, - {MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.DMP, calculateDraftDmps(false), MetricLabels.DRAFT); + this.setGaugeValue(gauges, MetricNames.DMP, calculateFinalizedDmps(false), MetricLabels.FINALIZED); + this.setGaugeValue(gauges, MetricNames.DMP, calculatePublishedDmps(false), MetricLabels.PUBLISHED); + this.setGaugeValue(gauges, MetricNames.DMP, calculateDoiedDmps(false), MetricLabels.DOIED); - {MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateDraftDmps(true), MetricLabels.DRAFT); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateFinalizedDmps(true), MetricLabels.FINALIZED); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculatePublishedDmps(true), MetricLabels.PUBLISHED); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DMP, calculateDoiedDmps(true), MetricLabels.DOIED); - {MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.FUNDERS, calculateAllFunders(false), null); + this.setGaugeValue(gauges, MetricNames.GRANTS, calculateAllGrants(false), null); + this.setGaugeValue(gauges, MetricNames.PROJECTS, calculateAllProjects(false), null); + this.setGaugeValue(gauges, MetricNames.RESEARCHERS, calculateAllResearchers(false), null); - {MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.FUNDERS, calculateAllFunders(true), null); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.GRANTS, calculateAllGrants(true), null); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.PROJECTS, calculateAllProjects(true), null); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.RESEARCHERS, calculateAllResearchers(true), null); - {MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.DATASET, calculateDraftDatasets(false), MetricLabels.DRAFT); + this.setGaugeValue(gauges, MetricNames.DATASET, calculateFinalizedDatasets(false), MetricLabels.FINALIZED); + this.setGaugeValue(gauges, MetricNames.DATASET, calculatePublishedDatasets(false), MetricLabels.PUBLISHED); + this.setGaugeValue(gauges, MetricNames.DATASET, calculateDoiedDatasets(false), MetricLabels.DOIED); - {MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET, calculateDraftDatasets(true), MetricLabels.DRAFT); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET, calculateFinalizedDatasets(true), MetricLabels.FINALIZED); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET, calculatePublishedDatasets(true), MetricLabels.PUBLISHED); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET, calculateDoiedDatasets(true), MetricLabels.DOIED); - {MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.DATASET_TEMPLATE, calculateDraftTemplates(false), MetricLabels.DRAFT); + this.setGaugeValue(gauges, MetricNames.DATASET_TEMPLATE, calculateFinalizedTemplates(false), MetricLabels.ACTIVE); + this.setGaugeValue(gauges, MetricNames.DATASET_TEMPLATE, calculateUsedTemplates(false), MetricLabels.USED); - {MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, calculateDraftTemplates(true), MetricLabels.DRAFT); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, calculateFinalizedTemplates(true), MetricLabels.ACTIVE); + this.setGaugeValue(gauges, MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, calculateUsedTemplates(true), MetricLabels.USED); - {MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, + this.setGaugeValue(gauges, MetricNames.INSTALLATIONS, 1d, null); + } finally { + this.entityManager.enableTenantFilters(); + } + } - {MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())}, - {MetricNames.NEXUS + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())}, + @Override + public Map gaugesBuild() { + registry.clear(); - }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1])); - } - - private double calculateFinalizedDmps(){ - return this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).isActive(IsActive.Active).count(); - } + return Stream.of(new Object[][]{ + {MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())}, + {MetricNames.NEXUS_PREFIX + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, - private void setGauseValue(Map gauges, String name, Double amount, String label) { - if(label != null) { - gauges.get(name).labels(label).set(amount); - } else { - gauges.get(name).set(amount); - } - } + {MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())}, + {MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())}, + {MetricNames.PROJECTS, Gauge.build().name(MetricNames.PROJECTS).help("Number of registered Projects").register(registry.getPrometheusRegistry())}, + {MetricNames.RESEARCHERS, Gauge.build().name(MetricNames.RESEARCHERS).help("Number of Collaborators/Researchers").register(registry.getPrometheusRegistry())}, + + {MetricNames.NEXUS_PREFIX + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())}, + {MetricNames.NEXUS_PREFIX + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())}, + {MetricNames.NEXUS_PREFIX + MetricNames.PROJECTS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.PROJECTS).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())}, + {MetricNames.NEXUS_PREFIX + MetricNames.RESEARCHERS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.RESEARCHERS).help("Number of Collaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())}, + + {MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())}, + {MetricNames.NEXUS_PREFIX + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, + + {MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())}, + {MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, + + {MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())}, + + {MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())}, + + {MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())}, + {MetricNames.NEXUS_PREFIX + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, + + {MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())}, + {MetricNames.NEXUS_PREFIX + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS_PREFIX + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())}, + }).collect(Collectors.toMap(data -> (String) data[0], data -> (Gauge) data[1])); + } + + private double calculateDraftDmps(boolean forNexus) { + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Draft).isActive(IsActive.Active); + if (forNexus) + dmpQuery.after(_config.getNexusDate()); + return dmpQuery.count(); + } + + private double calculateFinalizedDmps(boolean forNexus) { + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).isActive(IsActive.Active); + if (forNexus) + dmpQuery.after(_config.getNexusDate()); + return dmpQuery.count(); + } + + private double calculatePublishedDmps(boolean forNexus) { + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).accessTypes(DmpAccessType.Public).isActive(IsActive.Active); + if (forNexus) + dmpQuery.after(_config.getNexusDate()); + return dmpQuery.count(); + } + + private double calculateDoiedDmps(boolean forNexus) { + EntityDoiQuery entityDoiQuery = this.queryFactory.query(EntityDoiQuery.class).types(EntityType.DMP).isActive(IsActive.Active); + if (forNexus) + entityDoiQuery.after(_config.getNexusDate()); + return entityDoiQuery.count(); + } + + private double calculateAllFunders(boolean forNexus) { + ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getFunderIds()).isActive(IsActive.Active); + if (forNexus) + referenceQuery.after(_config.getNexusDate()); + return referenceQuery.count(); + } + + private double calculateAllGrants(boolean forNexus) { + ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getGrantIds()).isActive(IsActive.Active); + if (forNexus) + referenceQuery.after(_config.getNexusDate()); + return referenceQuery.count(); + } + + private double calculateAllProjects(boolean forNexus) { + ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getProjectIds()).isActive(IsActive.Active); + if (forNexus) + referenceQuery.after(_config.getNexusDate()); + return referenceQuery.count(); + } + + private double calculateAllResearchers(boolean forNexus) { + ReferenceQuery referenceQuery = this.queryFactory.query(ReferenceQuery.class).typeIds(_config.getReferenceTypes().getResearcherIds()).isActive(IsActive.Active); + if (forNexus) + referenceQuery.after(_config.getNexusDate()); + return referenceQuery.count(); + } + + private double calculateDraftDatasets(boolean forNexus) { + DescriptionQuery descriptionQuery = this.queryFactory.query(DescriptionQuery.class).statuses(DescriptionStatus.Draft).isActive(IsActive.Active); + if (forNexus) + descriptionQuery.createdAfter(_config.getNexusDate()); + return descriptionQuery.count(); + } + + private double calculateFinalizedDatasets(boolean forNexus) { + DescriptionQuery descriptionQuery = this.queryFactory.query(DescriptionQuery.class).statuses(DescriptionStatus.Finalized).isActive(IsActive.Active); + if (forNexus) + descriptionQuery.createdAfter(_config.getNexusDate()); + return descriptionQuery.count(); + } + + private double calculatePublishedDatasets(boolean forNexus) { + DescriptionQuery descriptionQuery = this.queryFactory.query(DescriptionQuery.class).statuses(DescriptionStatus.Finalized).isActive(IsActive.Active); + if (forNexus) + descriptionQuery.createdAfter(_config.getNexusDate()); + DmpQuery dmpQuery = this.queryFactory.query(DmpQuery.class).isActive(IsActive.Active).statuses(DmpStatus.Finalized).accessTypes(DmpAccessType.Public); + descriptionQuery.dmpSubQuery(dmpQuery); + return descriptionQuery.count(); + } + + private double calculateDoiedDatasets(boolean forNexus) { + return 0; + } + + private double calculateDraftTemplates(boolean forNexus) { + DescriptionTemplateQuery descriptionTemplateQuery = this.queryFactory.query(DescriptionTemplateQuery.class).statuses(DescriptionTemplateStatus.Draft).isActive(IsActive.Active); + if (forNexus) + descriptionTemplateQuery.after(_config.getNexusDate()); + return descriptionTemplateQuery.count(); + } + + private double calculateFinalizedTemplates(boolean forNexus) { + DescriptionTemplateQuery descriptionTemplateQuery = this.queryFactory.query(DescriptionTemplateQuery.class).statuses(DescriptionTemplateStatus.Finalized).isActive(IsActive.Active); + if (forNexus) + descriptionTemplateQuery.after(_config.getNexusDate()); + return descriptionTemplateQuery.count(); + } + + private double calculateUsedTemplates(boolean forNexus) { + DmpDescriptionTemplateQuery dmpDescriptionTemplateQuery = this.queryFactory.query(DmpDescriptionTemplateQuery.class).isActive(IsActive.Active); + dmpDescriptionTemplateQuery.setDistinct(true); + if (forNexus) + dmpDescriptionTemplateQuery.after(_config.getNexusDate()); + return dmpDescriptionTemplateQuery.count(); + } + + private void setGaugeValue(Map gauges, String name, Double amount, String label) { + if (label != null) { + gauges.get(name).labels(label).set(amount); + } else { + gauges.get(name).set(amount); + } + } } diff --git a/dmp-backend/core/src/main/java/eu/eudat/service/metrics/UpdateMetricsTaskProperties.java b/dmp-backend/core/src/main/java/eu/eudat/service/metrics/UpdateMetricsTaskProperties.java index 8ee52bba7..990bf13fa 100644 --- a/dmp-backend/core/src/main/java/eu/eudat/service/metrics/UpdateMetricsTaskProperties.java +++ b/dmp-backend/core/src/main/java/eu/eudat/service/metrics/UpdateMetricsTaskProperties.java @@ -1,27 +1,96 @@ package eu.eudat.service.metrics; - import org.springframework.boot.context.properties.ConfigurationProperties; +import java.time.Instant; +import java.util.List; +import java.util.UUID; + @ConfigurationProperties(prefix = "metrics.task") public class UpdateMetricsTaskProperties { - private boolean enable; - private int intervalSeconds; - public boolean getEnable() { - return enable; - } + private boolean enable; - public void setEnable(boolean enable) { - this.enable = enable; - } + private int intervalSeconds; - public int getIntervalSeconds() { - return intervalSeconds; - } + private Instant nexusDate; + + private ReferenceTypesProperties referenceTypes; + + public boolean getEnable() { + return enable; + } + + public int getIntervalSeconds() { + return intervalSeconds; + } + + public Instant getNexusDate() { + return nexusDate; + } + + public ReferenceTypesProperties getReferenceTypes() { + return referenceTypes; + } + + public void setEnable(boolean enable) { + this.enable = enable; + } + + public void setIntervalSeconds(int intervalSeconds) { + this.intervalSeconds = intervalSeconds; + } + + public void setNexusDate(Instant nexusDate) { + this.nexusDate = nexusDate; + } + + public void setReferenceTypes(ReferenceTypesProperties referenceTypes) { + this.referenceTypes = referenceTypes; + } + + public static class ReferenceTypesProperties { + + private List funderIds; + + private List grantIds; + + private List projectIds; + + private List researcherIds; + + public List getFunderIds() { + return funderIds; + } + + public List getGrantIds() { + return grantIds; + } + + public List getProjectIds() { + return projectIds; + } + + public List getResearcherIds() { + return researcherIds; + } + + public void setFunderIds(List funderIds) { + this.funderIds = funderIds; + } + + public void setGrantIds(List grantIds) { + this.grantIds = grantIds; + } + + public void setProjectIds(List projectIds) { + this.projectIds = projectIds; + } + + public void setResearcherIds(List researcherIds) { + this.researcherIds = researcherIds; + } + } - public void setIntervalSeconds(int intervalSeconds) { - this.intervalSeconds = intervalSeconds; - } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java index ef82532fe..a41c9b360 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java @@ -1,6 +1,7 @@ package eu.eudat.logic.managers; import eu.eudat.commons.enums.DescriptionTemplateStatus; +import eu.eudat.commons.metrics.MetricLabels; import eu.eudat.commons.metrics.MetricNames; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.Gauge; @@ -29,8 +30,8 @@ public class MetricsManager { private final Map gauges; public static final Map datasetTemplateStatus = Stream.of(new Object[][] { - { DescriptionTemplateStatus.Draft.getValue(), MetricNames.DRAFT }, - { DescriptionTemplateStatus.Finalized.getValue(), MetricNames.ACTIVE }, + { DescriptionTemplateStatus.Draft.getValue(), MetricLabels.DRAFT }, + { DescriptionTemplateStatus.Finalized.getValue(), MetricLabels.ACTIVE }, }).collect(Collectors.toMap(data -> (Short) data[0], data -> (String) data[1])); public void increaseValue(String name, int amount, String label) { diff --git a/dmp-backend/web/src/main/resources/config/metrics.yml b/dmp-backend/web/src/main/resources/config/metrics.yml index b059adca9..0ad1ad292 100644 --- a/dmp-backend/web/src/main/resources/config/metrics.yml +++ b/dmp-backend/web/src/main/resources/config/metrics.yml @@ -2,6 +2,12 @@ metrics: task: enable: true intervalSeconds: 600 + nexusDate: "2021-01-01T00:00:00.00Z" + referenceTypes: + funderIds: ["538928bb-c7c6-452e-b66d-08e539f5f082"] + grantIds: ["5b9c284f-f041-4995-96cc-fad7ad13289c"] + projectIds: ["3d372db5-a456-45e6-a845-e41e1a8311f8"] + researcherIds: ["5a2112e7-ea99-4cfe-98a1-68665e26726e"] management: endpoint: metrics: @@ -13,7 +19,7 @@ management: web: base-path: / exposure: - include: ["prometheus","health","metrics" ] + include: ["prometheus","health","metrics"] metrics: enabled: true