argos/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java

414 lines
22 KiB
Java

package eu.eudat.logic.managers;
import eu.eudat.data.dao.criteria.*;
import eu.eudat.data.entities.DatasetProfile;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.types.MetricNames;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.Gauge;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import javax.transaction.Transactional;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
public class MetricsManager {
private final static Logger logger = LoggerFactory.getLogger(MetricsManager.class);
private final Map<String, Gauge> gauges;
public static final Map<Short, String> datasetTemplateStatus = Stream.of(new Object[][] {
{ DatasetProfile.Status.SAVED.getValue(), MetricNames.DRAFT },
{ DatasetProfile.Status.FINALIZED.getValue(), MetricNames.ACTIVE },
}).collect(Collectors.toMap(data -> (Short) data[0], data -> (String) data[1]));
public void increaseValue(String name, int amount, String label) {
if(label != null) {
gauges.get(name).labels(label).inc(amount);
} else {
gauges.get(name).inc(amount);
}
}
public void decreaseValue(String name, int amount, String label) {
if(label != null) {
gauges.get(name).labels(label).dec(amount);
} else {
gauges.get(name).dec(amount);
}
}
public Integer getValue(String name, String label) {
if(label != null) {
return Double.valueOf(gauges.get(name).labels(label).get()).intValue();
} else {
return Double.valueOf(gauges.get(name).get()).intValue();
}
}
public void calculateValue(String name, int amount, String label) {
Integer orig = getValue(name, label);
int diff = orig - amount;
if (diff != 0) {
if (diff > 0) {
decreaseValue(name, diff, label);
} else {
increaseValue(name, Math.abs(diff), label);
}
}
}
private final ApiContext apiContext;
private final UserManager userManager;
private final Environment environment;
@Autowired
public MetricsManager(ApiContext apiContext, UserManager userManager, Environment environment, PrometheusMeterRegistry registry) {
this.apiContext = apiContext;
this.userManager = userManager;
this.environment = environment;
registry.clear();
this.gauges = Stream.of( new Object[][]{
{MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())},
{MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())},
{MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())},
{MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())},
{MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())},
{MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.INSTALLATIONS, Gauge.build().name(MetricNames.NEXUS + MetricNames.INSTALLATIONS).help("Number of Installations").register(registry.getPrometheusRegistry())},
}).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
}
@PostConstruct
@Transactional
@Scheduled(initialDelay = 1000 * 60 * 60, fixedDelay = 1000 * 60 * 60)
public void init() throws IOException {
logger.info("Start calculating Metrics");
calculateValue(MetricNames.DMP, (int) countAllDraftDMPs(), MetricNames.DRAFT);
calculateValue(MetricNames.DMP, (int) countAllFinalizedDMPs(), MetricNames.FINALIZED);
calculateValue(MetricNames.DMP, (int) countAllPublishedDMPs(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DMP, (int) countAllDoiedDMPs(), MetricNames.DOIED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDraftDMPs(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllFinalizedDMPs(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllPublishedDMPs(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDoiedDMPs(true), MetricNames.DOIED);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(), MetricNames.DRAFT);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(), MetricNames.FINALIZED);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(), MetricNames.DOIED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(true), MetricNames.DOIED);
calculateValue(MetricNames.FUNDERS, (int) countAllFunders(), null);
calculateValue(MetricNames.GRANTS, (int) countAllGrants(), null);
calculateValue(MetricNames.PROJECT, (int) countAllProjects(), null);
calculateValue(MetricNames.RESEARCHER, (int) countAllResearchers(), null);
calculateValue(MetricNames.NEXUS + MetricNames.FUNDERS, (int) countAllFunders(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.GRANTS, (int) countAllGrants(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.PROJECT, (int) countAllProjects(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.RESEARCHER, (int) countAllResearchers(true), null);
calculateValue(MetricNames.DATASET, (int) countAllDraftDatasets(), MetricNames.DRAFT);
calculateValue(MetricNames.DATASET, (int) countAllFinalizedDatasets(), MetricNames.FINALIZED);
calculateValue(MetricNames.DATASET, (int) countAllPublicDatasets(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DATASET, (int) countAllDatasetsWithDoi(), MetricNames.DOIED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDraftDatasets(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllFinalizedDatasets(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllPublicDatasets(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDatasetsWithDoi(true), MetricNames.DOIED);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(), MetricNames.DRAFT);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(), MetricNames.ACTIVE);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(), MetricNames.USED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(true), MetricNames.ACTIVE);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(true), MetricNames.USED);
calculateValue(MetricNames.USERS, (int) userManager.countActiveUsers().intValue(), MetricNames.LOGGEDIN);
calculateValue(MetricNames.USERS, (int) userManager.countAllUsers().intValue(), MetricNames.TOTAL);
try (Stream<Path> paths = Files.list(Paths.get(Objects.requireNonNull(this.environment.getProperty("userguide.path"))))) {
long files = paths.count();
calculateValue(MetricNames.LANGUAGES, (int) files, null);
}
calculateValue(MetricNames.INSTALLATIONS, 1, null);
calculateValue(MetricNames.NEXUS + MetricNames.INSTALLATIONS, 1, null);
logger.info("Metrics calculation Completed");
}
private Date getNexusDate() {
try {
return new SimpleDateFormat("yyyy-MM-dd").parse("2021-01-01");
} catch (ParseException e) {
logger.error(e.getLocalizedMessage(), e);
}
return Date.from(LocalDate.of(2021, 1, 1).atStartOfDay(ZoneId.systemDefault()).toInstant());
}
private long countAllDraftDMPs(){
return countAllDraftDMPs(false);
}
private long countAllDraftDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllFinalizedDMPs() {
return countAllFinalizedDMPs(false);
}
private long countAllFinalizedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllPublishedDMPs() {
return countAllPublishedDMPs(false);
}
private long countAllPublishedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllDoiedDMPs() {
return countAllDoiedDMPs(false);
}
private long countAllDoiedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllDraftDMPsWithGrantId() {
return countAllDraftDMPsWithGrantId(false);
}
private long countAllDraftDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllFinalizedDMPsWithGrantId() {
return countAllFinalizedDMPsWithGrantId(false);
}
private long countAllFinalizedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllPublishedDMPsWithGrantId() {
return countAllPublishedDMPsWithGrantId(false);
}
private long countAllPublishedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllDoiedDMPsWithGrantId() {
return countAllDoiedDMPsWithGrantId(false);
}
private long countAllDoiedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllResearchers() {
return countAllResearchers(false);
}
private long countAllResearchers(boolean countNexus) {
ResearcherCriteria criteria = new ResearcherCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count();
}
private long countAllProjects() {
return countAllProjects(false);
}
private long countAllProjects(boolean countNexus) {
ProjectCriteria criteria = new ProjectCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count();
}
private long countAllFunders() {
return countAllFunders(false);
}
private long countAllFunders(boolean countNexus) {
FunderCriteria criteria = new FunderCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count();
}
private long countAllGrants() {
return countAllGrants(false);
}
private long countAllGrants(boolean countNexus) {
GrantCriteria criteria = new GrantCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count();
}
public long countAllDraftDatasets() {
return countAllDraftDatasets(false);
}
public long countAllDraftDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllFinalizedDatasets() {
return countAllFinalizedDatasets(false);
}
public long countAllFinalizedDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllPublicDatasets() {
return countAllPublicDatasets(false);
}
public long countAllPublicDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setIsPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllDatasetsWithDoi() {
return countAllDatasetsWithDoi(false);
}
public long countAllDatasetsWithDoi(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllDraftTemplates() {
return countAllDraftTemplates(false);
}
public long countAllDraftTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
public long countAllFinalizedTemplates() {
return countAllFinalizedTemplates(false);
}
public long countAllFinalizedTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
@Transactional
public long countAllUsedTemplates() {
return countAllUsedTemplates(false);
}
@Transactional
public long countAllUsedTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
criteria.setAllVersions(false);
if (countNexus) criteria.setPeriodStart(getNexusDate());
List<DatasetProfile> datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).withFields(Collections.singletonList("id")).toList();
DatasetCriteria datasetCriteria = new DatasetCriteria();
datasetCriteria.setDatasetTemplates(datasetProfiles.stream().map(DatasetProfile::getId).collect(Collectors.toList()));
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria).select(root -> root.getProfile().getId()).stream().distinct().count();
}
}