package eu.eudat.logic.managers; import eu.eudat.data.dao.criteria.*; import eu.eudat.data.entities.DatasetProfile; import eu.eudat.logic.services.ApiContext; import eu.eudat.types.MetricNames; import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.Gauge; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.env.Environment; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.transaction.Transactional; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.text.ParseException; import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.ZoneId; import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @Component public class MetricsManager { private final static Logger logger = LoggerFactory.getLogger(MetricsManager.class); private final Map gauges; public static final Map datasetTemplateStatus = Stream.of(new Object[][] { { DatasetProfile.Status.SAVED.getValue(), MetricNames.DRAFT }, { DatasetProfile.Status.FINALIZED.getValue(), MetricNames.ACTIVE }, }).collect(Collectors.toMap(data -> (Short) data[0], data -> (String) data[1])); public void increaseValue(String name, int amount, String label) { if(label != null) { gauges.get(name).labels(label).inc(amount); } else { gauges.get(name).inc(amount); } } public void decreaseValue(String name, int amount, String label) { if(label != null) { gauges.get(name).labels(label).dec(amount); } else { gauges.get(name).dec(amount); } } public Integer getValue(String name, String label) { if(label != null) { return Double.valueOf(gauges.get(name).labels(label).get()).intValue(); } else { return Double.valueOf(gauges.get(name).get()).intValue(); } } public void calculateValue(String name, int amount, String label) { Integer orig = getValue(name, label); int diff = orig - amount; if (diff != 0) { if (diff > 0) { decreaseValue(name, diff, label); } else { increaseValue(name, Math.abs(diff), label); } } } private final ApiContext apiContext; private final UserManager userManager; private final Environment environment; @Autowired public MetricsManager(ApiContext apiContext, UserManager userManager, Environment environment, PrometheusMeterRegistry registry) { this.apiContext = apiContext; this.userManager = userManager; this.environment = environment; registry.clear(); this.gauges = Stream.of( new Object[][]{ {MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())}, {MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())}, {MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())}, {MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())}, {MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())}, {MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())}, {MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}, {MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())}, {MetricNames.NEXUS + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())}, }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1])); } @PostConstruct @Transactional @Scheduled(initialDelay = 1000 * 60 * 60, fixedDelay = 1000 * 60 * 60) public void init() throws IOException { logger.info("Start calculating Metrics"); calculateValue(MetricNames.DMP, (int) countAllDraftDMPs(), MetricNames.DRAFT); calculateValue(MetricNames.DMP, (int) countAllFinalizedDMPs(), MetricNames.FINALIZED); calculateValue(MetricNames.DMP, (int) countAllPublishedDMPs(), MetricNames.PUBLISHED); calculateValue(MetricNames.DMP, (int) countAllDoiedDMPs(), MetricNames.DOIED); calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDraftDMPs(true), MetricNames.DRAFT); calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllFinalizedDMPs(true), MetricNames.FINALIZED); calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllPublishedDMPs(true), MetricNames.PUBLISHED); calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDoiedDMPs(true), MetricNames.DOIED); calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(), MetricNames.DRAFT); calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(), MetricNames.FINALIZED); calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(), MetricNames.PUBLISHED); calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(), MetricNames.DOIED); calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(true), MetricNames.DRAFT); calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(true), MetricNames.FINALIZED); calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(true), MetricNames.PUBLISHED); calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(true), MetricNames.DOIED); calculateValue(MetricNames.FUNDERS, (int) countAllFunders(), null); calculateValue(MetricNames.GRANTS, (int) countAllGrants(), null); calculateValue(MetricNames.PROJECT, (int) countAllProjects(), null); calculateValue(MetricNames.RESEARCHER, (int) countAllResearchers(), null); calculateValue(MetricNames.NEXUS + MetricNames.FUNDERS, (int) countAllFunders(true), null); calculateValue(MetricNames.NEXUS + MetricNames.GRANTS, (int) countAllGrants(true), null); calculateValue(MetricNames.NEXUS + MetricNames.PROJECT, (int) countAllProjects(true), null); calculateValue(MetricNames.NEXUS + MetricNames.RESEARCHER, (int) countAllResearchers(true), null); calculateValue(MetricNames.DATASET, (int) countAllDraftDatasets(), MetricNames.DRAFT); calculateValue(MetricNames.DATASET, (int) countAllFinalizedDatasets(), MetricNames.FINALIZED); calculateValue(MetricNames.DATASET, (int) countAllPublicDatasets(), MetricNames.PUBLISHED); calculateValue(MetricNames.DATASET, (int) countAllDatasetsWithDoi(), MetricNames.DOIED); calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDraftDatasets(true), MetricNames.DRAFT); calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllFinalizedDatasets(true), MetricNames.FINALIZED); calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllPublicDatasets(true), MetricNames.PUBLISHED); calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDatasetsWithDoi(true), MetricNames.DOIED); calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(), MetricNames.DRAFT); calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(), MetricNames.ACTIVE); calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(), MetricNames.USED); calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(true), MetricNames.DRAFT); calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(true), MetricNames.ACTIVE); calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(true), MetricNames.USED); calculateValue(MetricNames.USERS, (int) userManager.countActiveUsers().intValue(), MetricNames.LOGGEDIN); calculateValue(MetricNames.USERS, (int) userManager.countAllUsers().intValue(), MetricNames.TOTAL); try (Stream paths = Files.list(Paths.get(Objects.requireNonNull(this.environment.getProperty("userguide.path"))))) { long files = paths.count(); calculateValue(MetricNames.LANGUAGES, (int) files, null); } calculateValue(MetricNames.INSTALLATIONS, 1, null); calculateValue(MetricNames.NEXUS + MetricNames.INSTALLATIONS, 1, null); logger.info("Metrics calculation Completed"); } private Date getNexusDate() { try { return new SimpleDateFormat("yyyy-MM-dd").parse("2021-01-01"); } catch (ParseException e) { logger.error(e.getLocalizedMessage(), e); } return Date.from(LocalDate.of(2021, 1, 1).atStartOfDay(ZoneId.systemDefault()).toInstant()); } private long countAllDraftDMPs(){ return countAllDraftDMPs(false); } private long countAllDraftDMPs(boolean countNexus) { DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setStatus(0); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); } private long countAllFinalizedDMPs() { return countAllFinalizedDMPs(false); } private long countAllFinalizedDMPs(boolean countNexus) { DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setStatus(1); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); } private long countAllPublishedDMPs() { return countAllPublishedDMPs(false); } private long countAllPublishedDMPs(boolean countNexus) { DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setIsPublic(true); criteria.setOnlyPublic(true); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); } private long countAllDoiedDMPs() { return countAllDoiedDMPs(false); } private long countAllDoiedDMPs(boolean countNexus) { DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setHasDoi(true); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); } private long countAllDraftDMPsWithGrantId() { return countAllDraftDMPsWithGrantId(false); } private long countAllDraftDMPsWithGrantId(boolean countNexus) { DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setStatus(0); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count(); } private long countAllFinalizedDMPsWithGrantId() { return countAllFinalizedDMPsWithGrantId(false); } private long countAllFinalizedDMPsWithGrantId(boolean countNexus) { DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setStatus(1); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count(); } private long countAllPublishedDMPsWithGrantId() { return countAllPublishedDMPsWithGrantId(false); } private long countAllPublishedDMPsWithGrantId(boolean countNexus) { DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setIsPublic(true); criteria.setOnlyPublic(true); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count(); } private long countAllDoiedDMPsWithGrantId() { return countAllDoiedDMPsWithGrantId(false); } private long countAllDoiedDMPsWithGrantId(boolean countNexus) { DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setHasDoi(true); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count(); } private long countAllResearchers() { return countAllResearchers(false); } private long countAllResearchers(boolean countNexus) { ResearcherCriteria criteria = new ResearcherCriteria(); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count(); } private long countAllProjects() { return countAllProjects(false); } private long countAllProjects(boolean countNexus) { ProjectCriteria criteria = new ProjectCriteria(); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count(); } private long countAllFunders() { return countAllFunders(false); } private long countAllFunders(boolean countNexus) { FunderCriteria criteria = new FunderCriteria(); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count(); } private long countAllGrants() { return countAllGrants(false); } private long countAllGrants(boolean countNexus) { GrantCriteria criteria = new GrantCriteria(); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count(); } public long countAllDraftDatasets() { return countAllDraftDatasets(false); } public long countAllDraftDatasets(boolean countNexus) { eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); criteria.setStatus(0); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); } public long countAllFinalizedDatasets() { return countAllFinalizedDatasets(false); } public long countAllFinalizedDatasets(boolean countNexus) { eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); criteria.setStatus(1); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); } public long countAllPublicDatasets() { return countAllPublicDatasets(false); } public long countAllPublicDatasets(boolean countNexus) { eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); criteria.setIsPublic(true); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); } public long countAllDatasetsWithDoi() { return countAllDatasetsWithDoi(false); } public long countAllDatasetsWithDoi(boolean countNexus) { eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); criteria.setHasDoi(true); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); } public long countAllDraftTemplates() { return countAllDraftTemplates(false); } public long countAllDraftTemplates(boolean countNexus) { DatasetProfileCriteria criteria = new DatasetProfileCriteria(); criteria.setStatus(0); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); } public long countAllFinalizedTemplates() { return countAllFinalizedTemplates(false); } public long countAllFinalizedTemplates(boolean countNexus) { DatasetProfileCriteria criteria = new DatasetProfileCriteria(); criteria.setStatus(1); if (countNexus) criteria.setPeriodStart(getNexusDate()); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); } @Transactional public long countAllUsedTemplates() { return countAllUsedTemplates(false); } @Transactional public long countAllUsedTemplates(boolean countNexus) { DatasetProfileCriteria criteria = new DatasetProfileCriteria(); criteria.setStatus(1); criteria.setAllVersions(false); if (countNexus) criteria.setPeriodStart(getNexusDate()); List datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).withFields(Collections.singletonList("id")).toList(); DatasetCriteria datasetCriteria = new DatasetCriteria(); datasetCriteria.setDatasetTemplates(datasetProfiles.stream().map(DatasetProfile::getId).collect(Collectors.toList())); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria).select(root -> root.getProfile().getId()).stream().distinct().count(); } }