Merge remote-tracking branch 'origin/dmp-refactoring' into dmp-refactoring

This commit is contained in:
amentis 2024-04-11 17:31:42 +03:00
commit e0317e95b8
10 changed files with 336 additions and 36 deletions

View File

@ -1,4 +1,4 @@
package eu.eudat.types;
package eu.eudat.commons.metrics;
public class MetricNames {
public static final String DATASET_TEMPLATE = "argos_dmp_templates";

View File

@ -0,0 +1,11 @@
package eu.eudat.service.metrics;
import io.prometheus.client.Gauge;
import javax.management.InvalidApplicationException;
import java.util.Map;
public interface MetricsService {
void calculate(Map<String, Gauge> gauges) throws InvalidApplicationException;
Map<String, Gauge> gaugesBuild();
}

View File

@ -0,0 +1,92 @@
package eu.eudat.service.metrics;
import eu.eudat.commons.enums.DmpStatus;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.metrics.MetricNames;
import eu.eudat.data.TenantEntityManager;
import eu.eudat.query.DmpQuery;
import gr.cite.tools.data.query.QueryFactory;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.Gauge;
import org.springframework.stereotype.Service;
import javax.management.InvalidApplicationException;
import java.util.Map;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Service
public class MetricsServiceImpl implements MetricsService {
private final PrometheusMeterRegistry registry;
private final QueryFactory queryFactory;
private final TenantEntityManager entityManager;
public MetricsServiceImpl(PrometheusMeterRegistry registry, QueryFactory queryFactory, TenantEntityManager entityManager) {
this.registry = registry;
this.queryFactory = queryFactory;
this.entityManager = entityManager;
}
@Override
public void calculate(Map<String, Gauge> gauges) throws InvalidApplicationException {
try {
this.entityManager.disableTenantFilters();
this.setGauseValue(gauges, MetricNames.DMP, calculateFinalizedDmps(), MetricNames.FINALIZED);
}finally {
this.entityManager.enableTenantFilters();
}
}
@Override
public Map<String, Gauge> gaugesBuild() {
registry.clear();
return Stream.of( new Object[][]{
{MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())},
{MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())},
{MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())},
{MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())},
{MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())},
{MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
}).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
}
private double calculateFinalizedDmps(){
return this.queryFactory.query(DmpQuery.class).statuses(DmpStatus.Finalized).isActive(IsActive.Active).count();
}
private void setGauseValue(Map<String, Gauge> gauges, String name, Double amount, String label) {
if(label != null) {
gauges.get(name).labels(label).set(amount);
} else {
gauges.get(name).set(amount);
}
}
}

View File

@ -0,0 +1,130 @@
package eu.eudat.service.metrics;
import eu.eudat.commons.enums.IsActive;
import eu.eudat.commons.fake.FakeRequestScope;
import eu.eudat.commons.metrics.MetricNames;
import eu.eudat.commons.scope.tenant.TenantScope;
import eu.eudat.data.QueueInboxEntity;
import eu.eudat.data.StorageFileEntity;
import eu.eudat.data.TenantEntity;
import eu.eudat.data.TenantEntityManager;
import eu.eudat.integrationevent.inbox.EventProcessingStatus;
import eu.eudat.model.StorageFile;
import eu.eudat.model.Tenant;
import eu.eudat.query.QueueInboxQuery;
import eu.eudat.query.StorageFileQuery;
import eu.eudat.query.TenantQuery;
import eu.eudat.service.storage.StorageFileService;
import gr.cite.queueinbox.entity.QueueInboxStatus;
import gr.cite.queueinbox.repository.CandidateInfo;
import gr.cite.tools.data.query.Ordering;
import gr.cite.tools.data.query.QueryFactory;
import gr.cite.tools.logging.LoggerService;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.Gauge;
import jakarta.persistence.EntityManager;
import jakarta.persistence.EntityManagerFactory;
import jakarta.persistence.EntityTransaction;
import jakarta.persistence.OptimisticLockException;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.config.ConfigurableBeanFactory;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationListener;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Service;
import java.io.Closeable;
import java.io.IOException;
import java.time.Instant;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Service
@Scope(value = ConfigurableBeanFactory.SCOPE_SINGLETON)
public class UpdateMetricsTask implements Closeable, ApplicationListener<ApplicationReadyEvent> {
private static final LoggerService logger = new LoggerService(LoggerFactory.getLogger(UpdateMetricsTask.class));
private final UpdateMetricsTaskProperties _config;
private final ApplicationContext applicationContext;
private ScheduledExecutorService scheduler;
private Map<String, Gauge> gauges;
public UpdateMetricsTask(
UpdateMetricsTaskProperties config,
ApplicationContext applicationContext) {
this._config = config;
this.applicationContext = applicationContext;
this.gauges = null;
}
@Override
public void onApplicationEvent(ApplicationReadyEvent event) {
long intervalSeconds = this._config .getIntervalSeconds();
if (this._config .getEnable() && intervalSeconds > 0) {
logger.info("File clean up run in {} seconds", intervalSeconds);
scheduler = Executors.newScheduledThreadPool(1);
scheduler.scheduleAtFixedRate(this::process, 10, intervalSeconds, TimeUnit.SECONDS);
} else {
scheduler = null;
}
}
@Override
public void close() throws IOException {
if (scheduler != null) this.scheduler.close();
}
protected void process() {
if (!this._config.getEnable()) return;
try {
this.ensureGauges();
this.calculate();
} catch (Exception ex) {
logger.error("Problem processing file cleanups. Breaking for next interval", ex);
}
}
private void ensureGauges() {
if (this.gauges != null) return;
try (FakeRequestScope ignored = new FakeRequestScope()) {
MetricsService metricsService = this.applicationContext.getBean(MetricsService.class);
this.gauges = metricsService.gaugesBuild();
} catch (Exception ex) {
logger.error("Problem executing purge. rolling back any db changes and marking error. Continuing...", ex);
}
}
private void calculate() {
EntityManager entityManager = null;
try (FakeRequestScope ignored = new FakeRequestScope()) {
try {
EntityManagerFactory entityManagerFactory = this.applicationContext.getBean(EntityManagerFactory.class);
entityManager = entityManagerFactory.createEntityManager();
TenantEntityManager tenantEntityManager = this.applicationContext.getBean(TenantEntityManager.class);
tenantEntityManager.setEntityManager(entityManager);
TenantScope tenantScope = this.applicationContext.getBean(TenantScope.class);
tenantScope.setTenant(null, tenantScope.getDefaultTenantCode());
MetricsService metricsService = this.applicationContext.getBean(MetricsService.class);
metricsService.calculate(this.gauges);
} finally {
if (entityManager != null) entityManager.close();
}
} catch (Exception ex) {
logger.error("Problem executing purge. rolling back any db changes and marking error. Continuing...", ex);
}
}
}

View File

@ -0,0 +1,9 @@
package eu.eudat.service.metrics;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Configuration;
@Configuration
@EnableConfigurationProperties(UpdateMetricsTaskProperties.class)
public class UpdateMetricsTaskConfiguration {
}

View File

@ -0,0 +1,27 @@
package eu.eudat.service.metrics;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties(prefix = "metrics.task")
public class UpdateMetricsTaskProperties {
private boolean enable;
private int intervalSeconds;
public boolean getEnable() {
return enable;
}
public void setEnable(boolean enable) {
this.enable = enable;
}
public int getIntervalSeconds() {
return intervalSeconds;
}
public void setIntervalSeconds(int intervalSeconds) {
this.intervalSeconds = intervalSeconds;
}
}

View File

@ -1,7 +1,7 @@
package eu.eudat.logic.managers;
import eu.eudat.commons.enums.DescriptionTemplateStatus;
import eu.eudat.types.MetricNames;
import eu.eudat.commons.metrics.MetricNames;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.Gauge;
import jakarta.annotation.PostConstruct;
@ -75,37 +75,37 @@ public class MetricsManager {
public MetricsManager(PrometheusMeterRegistry registry) {
registry.clear();
this.gauges = Stream.of( new Object[][]{
{MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())},
{MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())},
{MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())},
{MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())},
{MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())},
{MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
// {MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())},
// {MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
//
// {MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())},
// {MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())},
// {MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())},
// {MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())},
//
// {MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())},
// {MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())},
// {MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())},
// {MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())},
//
// {MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())},
//
// {MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
//
// {MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())},
//
// {MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
//
// {MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())},
//
// {MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())},
//
// {MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())},
//
// {MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
//
// {MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
// {MetricNames.NEXUS + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
}).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));

View File

@ -32,6 +32,7 @@ spring:
optional:classpath:config/dashboard.yml[.yml], optional:classpath:config/dashboard-${spring.profiles.active}.yml[.yml], optional:file:../config/dashboard-${spring.profiles.active}.yml[.yml],
optional:classpath:config/transformer.yml[.yml], optional:classpath:config/transformer-${spring.profiles.active}.yml[.yml], optional:file:../config/transformer-${spring.profiles.active}.yml[.yml],
optional:classpath:config/authorization.yml[.yml], optional:classpath:config/authorization-${spring.profiles.active}.yml[.yml], optional:file:../config/authorization-${spring.profiles.active}.yml[.yml],
optional:classpath:config/metrics.yml[.yml], optional:classpath:config/metrics-${spring.profiles.active}.yml[.yml], optional:file:../config/metrics-${spring.profiles.active}.yml[.yml],
optional:classpath:config/lock.yml[.yml], optional:classpath:config/lock-${spring.profiles.active}.yml[.yml], optional:file:../config/lock-${spring.profiles.active}.yml[.yml]

View File

@ -0,0 +1,30 @@
metrics:
task:
enable: true
intervalSeconds: 600
management:
endpoint:
metrics:
enabled: true
prometheus:
enabled: true
health:
show-details: always
web:
base-path: /
exposure:
include: ["prometheus","health","metrics" ]
metrics:
enabled: true
export:
prometheus:
enabled: true
enable:
http: true
jvm: true
jdbc: true
tomcat: true
logback: true
hikaricp: true
cache: true

View File

@ -2,7 +2,7 @@ web:
security:
enabled: true
authorized-endpoints: [ api ]
allowed-endpoints: [ api/public, api/dmp/public, api/description/public, /api/supportive-material/public, api/language/public, api/contact-support/public, api/dashboard/public ]
allowed-endpoints: [ api/public, api/dmp/public, api/description/public, /api/supportive-material/public, api/language/public, api/contact-support/public, api/dashboard/public, prometheus, health, metrics ]
idp:
api-key:
enabled: false