Add nexus prometheus metrics

This commit is contained in:
George Kalampokis 2021-11-09 13:05:38 +02:00
parent 58ecadd79b
commit a7badfd9d5
10 changed files with 201 additions and 6 deletions

View File

@ -2,6 +2,8 @@ package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.DatasetProfile;
import java.util.Date;
import java.util.List;
import java.util.UUID;
@ -36,6 +38,7 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
private Integer status;
private Integer role;
private List<UUID> ids;
private Date periodStart;
public boolean getAllVersions() { return allVersions; }
public void setAllVersions(boolean allVersions) { this.allVersions = allVersions; }
@ -87,4 +90,12 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
public void setIds(List<UUID> ids) {
this.ids = ids;
}
public Date getPeriodStart() {
return periodStart;
}
public void setPeriodStart(Date periodStart) {
this.periodStart = periodStart;
}
}

View File

@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.Funder;
import java.util.Date;
public class FunderCriteria extends Criteria<Funder> {
private String reference;
private String exactReference;
private Date periodStart;
public String getReference() {
return reference;
@ -20,4 +23,12 @@ public class FunderCriteria extends Criteria<Funder> {
public void setExactReference(String exactReference) {
this.exactReference = exactReference;
}
public Date getPeriodStart() {
return periodStart;
}
public void setPeriodStart(Date periodStart) {
this.periodStart = periodStart;
}
}

View File

@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.Project;
import java.util.Date;
public class ProjectCriteria extends Criteria<Project> {
private String reference;
private String exactReference;
private Date periodStart;
public String getReference() {
return reference;
@ -20,4 +23,12 @@ public class ProjectCriteria extends Criteria<Project> {
public void setExactReference(String exactReference) {
this.exactReference = exactReference;
}
public Date getPeriodStart() {
return periodStart;
}
public void setPeriodStart(Date periodStart) {
this.periodStart = periodStart;
}
}

View File

@ -2,9 +2,12 @@ package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.Researcher;
import java.util.Date;
public class ResearcherCriteria extends Criteria<Researcher> {
private String name;
private String reference;
private Date periodStart;
public String getName() {
return name;
@ -21,4 +24,12 @@ public class ResearcherCriteria extends Criteria<Researcher> {
public void setReference(String reference) {
this.reference = reference;
}
public Date getPeriodStart() {
return periodStart;
}
public void setPeriodStart(Date periodStart) {
this.periodStart = periodStart;
}
}

View File

@ -66,6 +66,8 @@ public class DatasetProfileDaoImpl extends DatabaseAccess<DatasetProfile> implem
} else {
query.where(((builder, root) -> builder.notEqual(root.get("status"), DatasetProfile.Status.DELETED.getValue())));
}
if (criteria.getPeriodStart() != null)
query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
return query;
}

View File

@ -31,6 +31,8 @@ public class FunderDaoImpl extends DatabaseAccess<Funder> implements FunderDao {
query.where((builder, root) -> builder.like(builder.upper(root.get("reference")), "%" + criteria.getReference().toUpperCase() + "%"));
if (criteria.getExactReference() != null)
query.where((builder, root) -> builder.like(builder.upper(root.get("reference")), criteria.getExactReference().toUpperCase()));
if (criteria.getPeriodStart() != null)
query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
query.where((builder, root) -> builder.notEqual(root.get("status"), Funder.Status.DELETED.getValue()));
return query;
}

View File

@ -30,6 +30,8 @@ public class ProjectDaoImpl extends DatabaseAccess<Project> implements ProjectDa
query.where((builder, root) -> builder.like(root.get("reference"), "%" + criteria.getReference() + "%"));
if (criteria.getExactReference() != null)
query.where((builder, root) -> builder.like(root.get("reference"), criteria.getExactReference()));
if (criteria.getPeriodStart() != null)
query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("startdate"), criteria.getPeriodStart()));
query.where((builder, root) -> builder.notEqual(root.get("status"), Project.Status.DELETED.getValue()));
return query;
}

View File

@ -29,6 +29,8 @@ public class ResearcherDaoImpl extends DatabaseAccess<Researcher> implements Res
query.where((builder, root) ->builder.or(builder.like(builder.lower(root.get("label")), "%" + criteria.getName().toLowerCase() + "%")));
if (criteria.getReference() != null && !criteria.getReference().isEmpty())
query.where((builder, root) ->builder.or(builder.like(root.get("reference"), criteria.getReference())));
if (criteria.getPeriodStart() != null)
query.where((builder, root) -> builder.greaterThanOrEqualTo(root.get("created"), criteria.getPeriodStart()));
return query;
}

View File

@ -19,10 +19,11 @@ import javax.transaction.Transactional;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.time.LocalDate;
import java.time.ZoneId;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -80,21 +81,33 @@ public class MetricsManager {
registry.clear();
this.gauges = Stream.of( new Object[][]{
{MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP).help("Number of managed DMPs during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register(registry.getPrometheusRegistry())},
{MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register(registry.getPrometheusRegistry())},
{MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register(registry.getPrometheusRegistry())},
{MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.FUNDERS, Gauge.build().name(MetricNames.NEXUS + MetricNames.FUNDERS).help("Number of registered Funders during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.GRANTS, Gauge.build().name(MetricNames.NEXUS + MetricNames.GRANTS).help("Number of registered Grants during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.PROJECT, Gauge.build().name(MetricNames.NEXUS + MetricNames.PROJECT).help("Number of registered Projects during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.RESEARCHER, Gauge.build().name(MetricNames.NEXUS + MetricNames.RESEARCHER).help("Number of Colaborators/Researchers during Nexus").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET).help("Number of managed Dataset Descriptions during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates during Nexus").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())},
{MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register(registry.getPrometheusRegistry())},
{MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())}
{MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, Gauge.build().name(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT).help("Number of Grants based on the status of the DMP that is using them during Nexus").labelNames("status").register(registry.getPrometheusRegistry())}
}).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
@ -110,25 +123,49 @@ public class MetricsManager {
calculateValue(MetricNames.DMP, (int) countAllPublishedDMPs(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DMP, (int) countAllDoiedDMPs(), MetricNames.DOIED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDraftDMPs(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllFinalizedDMPs(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllPublishedDMPs(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP, (int) countAllDoiedDMPs(true), MetricNames.DOIED);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(), MetricNames.DRAFT);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(), MetricNames.FINALIZED);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(), MetricNames.DOIED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDraftDMPsWithGrantId(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllFinalizedDMPsWithGrantId(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllPublishedDMPsWithGrantId(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DMP_WITH_GRANT, (int) countAllDoiedDMPsWithGrantId(true), MetricNames.DOIED);
calculateValue(MetricNames.FUNDERS, (int) countAllFunders(), null);
calculateValue(MetricNames.GRANTS, (int) countAllGrants(), null);
calculateValue(MetricNames.PROJECT, (int) countAllProjects(), null);
calculateValue(MetricNames.RESEARCHER, (int) countAllResearchers(), null);
calculateValue(MetricNames.NEXUS + MetricNames.FUNDERS, (int) countAllFunders(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.GRANTS, (int) countAllGrants(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.PROJECT, (int) countAllProjects(true), null);
calculateValue(MetricNames.NEXUS + MetricNames.RESEARCHER, (int) countAllResearchers(true), null);
calculateValue(MetricNames.DATASET, (int) countAllDraftDatasets(), MetricNames.DRAFT);
calculateValue(MetricNames.DATASET, (int) countAllFinalizedDatasets(), MetricNames.FINALIZED);
calculateValue(MetricNames.DATASET, (int) countAllPublicDatasets(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DATASET, (int) countAllDatasetsWithDoi(), MetricNames.DOIED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDraftDatasets(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllFinalizedDatasets(true), MetricNames.FINALIZED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllPublicDatasets(true), MetricNames.PUBLISHED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET, (int) countAllDatasetsWithDoi(true), MetricNames.DOIED);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(), MetricNames.DRAFT);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(), MetricNames.ACTIVE);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(), MetricNames.USED);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllDraftTemplates(true), MetricNames.DRAFT);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllFinalizedTemplates(true), MetricNames.ACTIVE);
calculateValue(MetricNames.NEXUS + MetricNames.DATASET_TEMPLATE, (int) countAllUsedTemplates(true), MetricNames.USED);
calculateValue(MetricNames.USERS, (int) userManager.countActiveUsers().intValue(), MetricNames.LOGGEDIN);
calculateValue(MetricNames.USERS, (int) userManager.countAllUsers().intValue(), MetricNames.TOTAL);
@ -138,117 +175,222 @@ public class MetricsManager {
logger.info("Metrics calculation Completed");
}
private long countAllDraftDMPs() {
private Date getNexusDate() {
try {
return new SimpleDateFormat("yyyy-MM-dd").parse("2021-01-01");
} catch (ParseException e) {
logger.error(e.getLocalizedMessage(), e);
}
return Date.from(LocalDate.of(2021, 1, 1).atStartOfDay(ZoneId.systemDefault()).toInstant());
}
private long countAllDraftDMPs(){
return countAllDraftDMPs(false);
}
private long countAllDraftDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllFinalizedDMPs() {
return countAllFinalizedDMPs(false);
}
private long countAllFinalizedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllPublishedDMPs() {
return countAllPublishedDMPs(false);
}
private long countAllPublishedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllDoiedDMPs() {
return countAllDoiedDMPs(false);
}
private long countAllDoiedDMPs(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllDraftDMPsWithGrantId() {
return countAllDraftDMPsWithGrantId(false);
}
private long countAllDraftDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllFinalizedDMPsWithGrantId() {
return countAllFinalizedDMPsWithGrantId(false);
}
private long countAllFinalizedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllPublishedDMPsWithGrantId() {
return countAllPublishedDMPsWithGrantId(false);
}
private long countAllPublishedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllDoiedDMPsWithGrantId() {
return countAllDoiedDMPsWithGrantId(false);
}
private long countAllDoiedDMPsWithGrantId(boolean countNexus) {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).groupBy((builder, root) -> root.get("grant")).count();
}
private long countAllResearchers() {
return countAllResearchers(false);
}
private long countAllResearchers(boolean countNexus) {
ResearcherCriteria criteria = new ResearcherCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count();
}
private long countAllProjects() {
return countAllProjects(false);
}
private long countAllProjects(boolean countNexus) {
ProjectCriteria criteria = new ProjectCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count();
}
private long countAllFunders() {
return countAllFunders(false);
}
private long countAllFunders(boolean countNexus) {
FunderCriteria criteria = new FunderCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count();
}
private long countAllGrants() {
return countAllGrants(false);
}
private long countAllGrants(boolean countNexus) {
GrantCriteria criteria = new GrantCriteria();
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count();
}
public long countAllDraftDatasets() {
return countAllDraftDatasets(false);
}
public long countAllDraftDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllFinalizedDatasets() {
return countAllFinalizedDatasets(false);
}
public long countAllFinalizedDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllPublicDatasets() {
return countAllPublicDatasets(false);
}
public long countAllPublicDatasets(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setIsPublic(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllDatasetsWithDoi() {
return countAllDatasetsWithDoi(false);
}
public long countAllDatasetsWithDoi(boolean countNexus) {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setHasDoi(true);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllDraftTemplates() {
return countAllDraftTemplates(false);
}
public long countAllDraftTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(0);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
public long countAllFinalizedTemplates() {
return countAllFinalizedTemplates(false);
}
public long countAllFinalizedTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
if (countNexus) criteria.setPeriodStart(getNexusDate());
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
@Transactional
public long countAllUsedTemplates() {
return countAllUsedTemplates(false);
}
@Transactional
public long countAllUsedTemplates(boolean countNexus) {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
criteria.setAllVersions(false);
if (countNexus) criteria.setPeriodStart(getNexusDate());
List<DatasetProfile> datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
List<DatasetProfile> filteredProfiles = new ArrayList<>();
for (DatasetProfile datasetProfile : datasetProfiles) {

View File

@ -19,4 +19,5 @@ public class MetricNames {
public static final String USED = "used";
public static final String LOGGEDIN = "loggedin";
public static final String TOTAL = "total";
public static final String NEXUS = "nexus_";
}