From f9814d33020702bfd016119caeccb8a0e75c1ded Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Tue, 29 Jun 2021 14:33:57 +0300 Subject: [PATCH 1/4] Update Spring boot framework to version 2.5.2 --- dmp-backend/pom.xml | 64 +++--- dmp-backend/web/pom.xml | 31 +-- .../main/java/eu/eudat/EuDatApplication.java | 2 +- .../java/eu/eudat/cache/ResponsesCache.java | 30 +-- .../DevelDatabaseConfiguration.java | 2 +- .../configurations/WebMVCConfiguration.java | 2 - .../main/java/eu/eudat/controllers/Admin.java | 8 +- .../main/java/eu/eudat/controllers/Login.java | 11 +- .../controllers/UserGuideController.java | 6 +- .../managers/DataManagementPlanManager.java | 87 ++------ .../eudat/logic/managers/DatasetManager.java | 34 +--- .../logic/managers/DatasetProfileManager.java | 29 +-- .../eudat/logic/managers/MetricsManager.java | 192 ++++++++++++++---- .../logic/proxy/fetching/RemoteFetcher.java | 2 +- .../resources/config/application.properties | 13 ++ 15 files changed, 287 insertions(+), 226 deletions(-) diff --git a/dmp-backend/pom.xml b/dmp-backend/pom.xml index 254577ace..be51b3cd1 100644 --- a/dmp-backend/pom.xml +++ b/dmp-backend/pom.xml @@ -10,7 +10,7 @@ org.springframework.boot spring-boot-starter-parent - 1.5.9.RELEASE + 2.5.2 @@ -27,25 +27,26 @@ 1.8 0.0.1-SNAPSHOT - 4.3.8.RELEASE + 5.3.8 - 4.2.3.RELEASE - 1.19.1 + 5.3.10.RELEASE + + + - 5.2.11.Final + 5.5.3.Final 1.9 4.11 1.2.17 1.7.12 - 9.0.7.v20131107 - + 1.2.3 - 1 - 3.0.1 - ikalyvas + + @@ -58,7 +59,7 @@ org.springframework spring-context-support - 4.0.0.RELEASE + ${org.springframework.version} org.springframework.boot @@ -139,7 +140,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.9.3 + 2.12.3 @@ -147,13 +148,13 @@ com.fasterxml.jackson.core jackson-core - 2.9.4 + 2.12.3 com.fasterxml.jackson.core jackson-databind - 2.9.4 + 2.12.3 @@ -257,36 +258,53 @@ 1.3.1 + + + javax.validation + validation-api + 2.0.1.Final + + + io.prometheus simpleclient - 0.10.0 + 0.11.0 io.prometheus simpleclient_hotspot - 0.10.0 + 0.11.0 io.prometheus simpleclient_httpserver - 0.10.0 + 0.11.0 io.prometheus simpleclient_pushgateway - 0.10.0 + 0.11.0 - + - io.prometheus - simpleclient_spring_boot - 0.10.0 + io.micrometer + micrometer-registry-prometheus + 1.7.1 + + + org.springframework.boot + spring-boot-starter-actuator + + + + io.micrometer + micrometer-core diff --git a/dmp-backend/web/pom.xml b/dmp-backend/web/pom.xml index d3cfd31e6..6798b6022 100644 --- a/dmp-backend/web/pom.xml +++ b/dmp-backend/web/pom.xml @@ -50,31 +50,37 @@ com.fasterxml.jackson.core jackson-annotations - 2.9.0 + 2.12.3 - + - + + + com.github.ben-manes.caffeine + caffeine + 2.9.0 + + - junit - junit - 4.12 + org.junit.jupiter + junit-jupiter-api + 5.7.2 test @@ -87,7 +93,7 @@ io.springfox springfox-swagger-ui - 2.7.0 + 3.0.0 @@ -101,7 +107,7 @@ org.springframework.social spring-social-linkedin - 1.0.0.RELEASE + 1.0.2.RELEASE @@ -143,7 +149,6 @@ org.springframework.boot spring-boot-maven-plugin - 1.5.9.RELEASE diff --git a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java index d9c2b65d6..31e87ddf5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java +++ b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java @@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.web.support.SpringBootServletInitializer; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; import org.springframework.scheduling.annotation.EnableAsync; @SpringBootApplication diff --git a/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java b/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java index 0a09a6731..427b61173 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java +++ b/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java @@ -1,12 +1,12 @@ package eu.eudat.cache; -import com.google.common.cache.CacheBuilder; +import com.github.benmanes.caffeine.cache.Caffeine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; -import org.springframework.cache.guava.GuavaCache; +import org.springframework.cache.caffeine.CaffeineCache; import org.springframework.cache.support.SimpleCacheManager; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; @@ -29,19 +29,19 @@ public class ResponsesCache { public CacheManager cacheManager() { logger.info("Loading ResponsesCache..."); SimpleCacheManager simpleCacheManager = new SimpleCacheManager(); - List caches = new ArrayList(); - caches.add(new GuavaCache("repositories", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("grants", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("projects", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("funders", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("externalDatasets", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("currencies", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("licenses", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + List caches = new ArrayList(); + caches.add(new CaffeineCache("repositories", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("grants", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("projects", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("funders", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("organisations", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("registries", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("services", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("tags", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("researchers", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("externalDatasets", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("currencies", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("licenses", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); simpleCacheManager.setCaches(caches); logger.info("OK"); return simpleCacheManager; diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java index 1c5c48bed..a1311f1f1 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java @@ -2,8 +2,8 @@ package eu.eudat.configurations; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; import org.springframework.context.annotation.*; import org.springframework.core.env.Environment; import org.springframework.dao.annotation.PersistenceExceptionTranslationPostProcessor; diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java index 2c64aa8d1..36d9c502c 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java @@ -4,7 +4,6 @@ import eu.eudat.controllers.interceptors.RequestInterceptor; import eu.eudat.logic.handlers.PrincipalArgumentResolver; import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.services.operations.authentication.AuthenticationService; -import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableAsync; @@ -18,7 +17,6 @@ import java.util.List; @EnableAsync @Configuration @EnableScheduling -@EnablePrometheusEndpoint public class WebMVCConfiguration extends WebMvcConfigurerAdapter { private ApiContext apiContext; diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java index 6a89b4429..8cee99ae8 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java @@ -44,13 +44,15 @@ public class Admin extends BaseController { private DatasetProfileManager datasetProfileManager; private UserManager userManager; private ConfigLoader configLoader; + private final MetricsManager metricsManager; @Autowired - public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader) { + public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader, MetricsManager metricsManager) { super(apiContext); this.datasetProfileManager = datasetProfileManager; this.userManager = userManager; this.configLoader = configLoader; + this.metricsManager = metricsManager; } @Transactional @@ -70,7 +72,7 @@ public class Admin extends BaseController { userDatasetProfile.setRole(0); getApiContext().getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile); datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile); - MetricsManager.increaseValue("argos_dataset_templates", 1, "total"); + metricsManager.increaseValue("argos_dataset_templates", 1, "total"); return ResponseEntity.status(HttpStatus.OK).body(modelDefinition.getId()); } @@ -89,7 +91,7 @@ public class Admin extends BaseController { eu.eudat.data.entities.DatasetProfile datasetProfile = this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(datasetprofile); datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile); if (datasetProfile.getStatus() == 1 && oldStatus == 0) { - MetricsManager.increaseValue("argos_dataset_templates", 1, "active"); + metricsManager.increaseValue("argos_dataset_templates", 1, "active"); } return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().status(ApiMessageCode.NO_MESSAGE)); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java index ac75578d9..a6172f284 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java @@ -64,6 +64,7 @@ public class Login { private ConfigurableProviderTokenValidator configurableProviderTokenValidator; private ZenodoTokenValidator zenodoTokenValidator; private ConfigLoader configLoader; + private final MetricsManager metricsManager; // private Logger logger; @@ -75,7 +76,8 @@ public class Login { B2AccessTokenValidator b2AccessTokenValidator, ORCIDTokenValidator orcidTokenValidator, LinkedInTokenValidator linkedInTokenValidator, OpenAIRETokenValidator openAIRETokenValidator, ConfigurableProviderTokenValidator configurableProviderTokenValidator, ZenodoTokenValidator zenodoTokenValidator, - ConfigLoader configLoader, UserManager userManager) { + ConfigLoader configLoader, UserManager userManager, + MetricsManager metricsManager) { this.customAuthenticationProvider = customAuthenticationProvider; this.nonVerifiedUserAuthenticationService = nonVerifiedUserAuthenticationService; this.twitterTokenValidator = twitterTokenValidator; @@ -87,6 +89,7 @@ public class Login { this.zenodoTokenValidator = zenodoTokenValidator; this.configLoader = configLoader; this.userManager = userManager; + this.metricsManager = metricsManager; } @@ -95,7 +98,7 @@ public class Login { public @ResponseBody ResponseEntity> externallogin(@RequestBody LoginInfo credentials) throws GeneralSecurityException, NullEmailException { logger.info("Trying To Login With " + credentials.getProvider()); - MetricsManager.increaseValue("argos_users", 1, "loggedin"); + metricsManager.increaseValue("argos_users", 1, "loggedin"); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(customAuthenticationProvider.authenticate(credentials)).status(ApiMessageCode.SUCCESS_MESSAGE)); } @@ -104,7 +107,7 @@ public class Login { public @ResponseBody ResponseEntity> nativelogin(@RequestBody Credentials credentials) throws NullEmailException { logger.info(credentials.getUsername() + " Trying To Login"); - MetricsManager.increaseValue("argos_users", 1, "loggedin"); + metricsManager.increaseValue("argos_users", 1, "loggedin"); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(userManager.authenticate(this.nonVerifiedUserAuthenticationService, credentials)).status(ApiMessageCode.SUCCESS_MESSAGE)); } @@ -165,7 +168,7 @@ public class Login { ResponseEntity> logout(Principal principal) { this.nonVerifiedUserAuthenticationService.Logout(principal.getToken()); logger.info(principal + " Logged Out"); - MetricsManager.decreaseValue("argos_users", 1, "loggedin"); + metricsManager.decreaseValue("argos_users", 1, "loggedin"); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().status(ApiMessageCode.NO_MESSAGE)); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java index 6cb588eef..811e0c4bd 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java @@ -30,16 +30,18 @@ import static eu.eudat.types.Authorities.ADMIN; public class UserGuideController { private Environment environment; + private final MetricsManager metricsManager; @Autowired - public UserGuideController(Environment environment) { + public UserGuideController(Environment environment, MetricsManager metricsManager) { this.environment = environment; + this.metricsManager = metricsManager; } @RequestMapping(path = "{lang}", method = RequestMethod.GET ) public ResponseEntity getUserGuide(@PathVariable(name = "lang") String lang) throws IOException { long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count(); - MetricsManager.calculateValue("argos_languages", (int) files, null); + metricsManager.calculateValue("argos_languages", (int) files, null); Stream walk = Files.walk(Paths.get(this.environment.getProperty("userguide.path"))); List result = walk.filter(Files::isRegularFile) .map(Path::toString).collect(Collectors.toList()); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java index 50782785a..9ffcc6dac 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java @@ -109,9 +109,10 @@ public class DataManagementPlanManager { private Environment environment; private RDAManager rdaManager; private UserManager userManager; + private final MetricsManager metricsManager; @Autowired - public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager) { + public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager, MetricsManager metricsManager) { this.apiContext = apiContext; this.datasetManager = datasetManager; this.utilitiesService = apiContext.getUtilitiesService(); @@ -119,6 +120,7 @@ public class DataManagementPlanManager { this.environment = environment; this.rdaManager = rdaManager; this.userManager = userManager; + this.metricsManager = metricsManager; } /* @@ -441,7 +443,7 @@ public class DataManagementPlanManager { setNotification = true; } else { - MetricsManager.increaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.increaseValue("argos_managed_dmps", 1, "draft"); } DMP newDmp = dataManagementPlan.toDataModel(); @@ -565,7 +567,7 @@ public class DataManagementPlanManager { if (dataManagementPlan.getStatus() == (int) DMP.DMPStatus.FINALISED.getValue() && dmp1.getStatus().equals(DMP.DMPStatus.FINALISED.getValue())) throw new Exception("DMP is finalized, therefore cannot be edited."); } else { - MetricsManager.increaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.increaseValue("argos_managed_dmps", 1, "draft"); } List datasets = new ArrayList<>(); DMP tempDMP = dataManagementPlan.toDataModel(); @@ -681,7 +683,7 @@ public class DataManagementPlanManager { newDmp.setDataset(new HashSet<>(databaseRepository.getDatasetDao().getWithCriteria(criteria1).toList())); this.updateIndex(newDmp); - MetricsManager.increaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.increaseValue("argos_managed_dmps", 1, "draft"); return newDmp.getId(); } @@ -695,16 +697,16 @@ public class DataManagementPlanManager { DMP oldDmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(uuid); switch (oldDmp.getStatus()) { case 0: - MetricsManager.decreaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "draft"); break; case 1: if (oldDmp.getDoi() != null) { - MetricsManager.decreaseValue("argos_managed_dmps", 1, "doied"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "doied"); } if (oldDmp.isPublic()) { - MetricsManager.decreaseValue("argos_managed_dmps", 1, "published"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "published"); } - MetricsManager.decreaseValue("argos_managed_dmps", 1, "finalized"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "finalized"); break; } oldDmp.setStatus(DMP.DMPStatus.DELETED.getValue()); @@ -744,7 +746,7 @@ public class DataManagementPlanManager { else { researcher.setCreationUser(user); researcherRepository.createOrUpdate(researcher); - MetricsManager.increaseValue("argos_researchers", 1, null); + metricsManager.increaseValue("argos_researchers", 1, null); } } } @@ -826,14 +828,14 @@ public class DataManagementPlanManager { project.setType(Project.ProjectType.EXTERNAL.getValue()); if (project.getId() == null) project.setId(UUID.randomUUID()); projectDao.createOrUpdate(project); - MetricsManager.increaseValue("argos_projects", 1, null); + metricsManager.increaseValue("argos_projects", 1, null); } } else { project.setType(Project.ProjectType.EXTERNAL.getValue()); if (project.getId() == null) project.setId(UUID.randomUUID()); projectDao.createOrUpdate(project); - MetricsManager.increaseValue("argos_projects", 1, null); + metricsManager.increaseValue("argos_projects", 1, null); } } } @@ -910,7 +912,7 @@ public class DataManagementPlanManager { return newDataset; }).thenApplyAsync(item -> { futures.add(datasetDao.createOrUpdateAsync(item).whenComplete(((dataset1, throwable) -> { - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); eu.eudat.elastic.entities.Dataset datasetElastic = new eu.eudat.elastic.entities.Dataset(); datasetElastic.setId(dataset1.getId().toString()); datasetElastic.setLabel(dataset1.getLabel()); @@ -969,7 +971,7 @@ public class DataManagementPlanManager { dmp.setPublic(true); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp); this.updateIndex(dmp); - MetricsManager.increaseValue("argos_managed_dmps", 1, "published"); + metricsManager.increaseValue("argos_managed_dmps", 1, "published"); DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setGroupIds(Collections.singletonList(dmp.getGroupId())); criteria.setAllVersions(true); @@ -982,7 +984,7 @@ public class DataManagementPlanManager { tags = elastic.getTags(); } this.datasetManager.updateTags(dataset, tags); - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published"); } catch (Exception e) { logger.error(e.getMessage(), e); } @@ -1068,11 +1070,11 @@ public class DataManagementPlanManager { this.updateIndex(dmp); UserInfo user = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal.getId()); sendNotification(dmp, user, NotificationType.DMP_FINALISED); - MetricsManager.decreaseValue("argos_managed_dmps", 1, "draft"); - MetricsManager.increaseValue("argos_managed_dmps", 1, "finalized"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.increaseValue("argos_managed_dmps", 1, "finalized"); this.updateDatasetsIndex(indexDatasets); - MetricsManager.decreaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "draft"); - MetricsManager.increaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "finalized"); + metricsManager.decreaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "draft"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "finalized"); } public void undoFinalize(UUID id, Principal principal) throws Exception { @@ -1084,8 +1086,8 @@ public class DataManagementPlanManager { dmp.setStatus(DMP.DMPStatus.ACTIVE.getValue()); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp); this.updateIndex(dmp); - MetricsManager.decreaseValue("argos_managed_dmps", 1, "finalized"); - MetricsManager.increaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "finalized"); + metricsManager.increaseValue("argos_managed_dmps", 1, "draft"); } public void updateUsers(UUID id, List users, Principal principal) throws Exception { @@ -2143,51 +2145,6 @@ public class DataManagementPlanManager { } - public long countAllDrafts() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllPublished() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setIsPublic(true); - criteria.setOnlyPublic(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllDoied() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setHasDoi(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllResearchers() { - ResearcherCriteria criteria = new ResearcherCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count(); - } - - public long countAllProjects() { - ProjectCriteria criteria = new ProjectCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count(); - } - - public long countAllFunders() { - FunderCriteria criteria = new FunderCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count(); - } - - public long countAllGrants() { - GrantCriteria criteria = new GrantCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count(); - } - /*public DataTableData getPublicPaged(DataManagmentPlanPublicTableRequest dataManagementPlanPublicTableRequest, String fieldsGroup, Principal principal) throws Exception { dataManagementPlanPublicTableRequest.setQuery(databaseRepository.getDmpDao().asQueryable().withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class))); QueryableList items = dataManagementPlanPublicTableRequest.applyCriteria(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java index 069e381e3..0e030783e 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java @@ -109,9 +109,10 @@ public class DatasetManager { private UserManager userManager; private ConfigLoader configLoader; private Environment environment; + private final MetricsManager metricsManager; @Autowired - public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment) { + public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment, MetricsManager metricsManager) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository(); @@ -119,6 +120,7 @@ public class DatasetManager { this.userManager = userManager; this.configLoader = configLoader; this.environment = environment; + this.metricsManager = metricsManager; } public DataTableData getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception { @@ -613,7 +615,7 @@ public class DatasetManager { sendNotification = true; } } else { - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); } if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null) throw new Exception("DMP is finalized, therefore Dataset cannot be edited."); @@ -857,7 +859,7 @@ public class DatasetManager { if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue()) throw new Exception("You cannot make public a Dataset That Has not Been Finalised"); datasetDao.createOrUpdate(dataset); - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published"); } public ResponseEntity getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException { @@ -946,7 +948,7 @@ public class DatasetManager { createServicesIfTheyDontExist(entity); createExternalDatasetsIfTheyDontExist(entity); - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity); } @@ -1102,28 +1104,4 @@ public class DatasetManager { listingModel.setProfileLatestVersion(islast); return listingModel; } - - public long countAllDraft() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllPublic() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setIsPublic(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllWithDoi() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setHasDoi(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java index 56e97cfb4..c145306b6 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java @@ -59,12 +59,14 @@ public class DatasetProfileManager { private DatabaseRepository databaseRepository; private Environment environment; private List cache; + private final MetricsManager metricsManager; @Autowired - public DatasetProfileManager(ApiContext apiContext, Environment environment) { + public DatasetProfileManager(ApiContext apiContext, Environment environment, MetricsManager metricsManager) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.environment = environment; + this.metricsManager = metricsManager; this.cache = new ArrayList<>(); } @@ -92,7 +94,7 @@ public class DatasetProfileManager { DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(UUID.fromString(id)); apiContext.getOperationsContext().getDatabaseRepository().detachEntity(profile); profile.setId(null); - MetricsManager.increaseValue("argos_dataset_templates", 1, "total"); + metricsManager.increaseValue("argos_dataset_templates", 1, "total"); return profile; } @@ -232,7 +234,7 @@ public class DatasetProfileManager { File localFile = convert(multiPartFile); eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileModel.DatasetProfile profile = xmlBuilder.build(localFile); Files.deleteIfExists(localFile.toPath()); - MetricsManager.increaseValue("argos_dataset_templates", 1, "total"); + metricsManager.increaseValue("argos_dataset_templates", 1, "total"); return profile; } catch (IOException e) { logger.error(e.getMessage(), e); @@ -311,25 +313,4 @@ public class DatasetProfileManager { }).collect(Collectors.toList())); } } - - public long countAlldraft() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); - } - - @Transactional - public long countAllUsed() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(1); - criteria.setAllVersions(false); - List datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList(); - return datasetProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).filter(datasetProfile -> !datasetProfile.getDataset().isEmpty()).count(); - } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java index 353171093..9dc432068 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java @@ -1,5 +1,11 @@ package eu.eudat.logic.managers; +import eu.eudat.data.dao.criteria.*; +import eu.eudat.data.entities.Dataset; +import eu.eudat.data.entities.DatasetProfile; +import eu.eudat.logic.services.ApiContext; +import eu.eudat.logic.utilities.helpers.StreamDistinctBy; +import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.Gauge; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,32 +19,16 @@ import javax.transaction.Transactional; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @Component public class MetricsManager { private final static Logger logger = LoggerFactory.getLogger(MetricsManager.class); - private static Map gauges = Stream.of( new Object[][]{ - {"argos_managed_dmps", Gauge.build().name("argos_managed_dmps").help("Number of managed DMPs").labelNames("status").register()}, - - {"argos_funders", Gauge.build().name("argos_funders").help("Number of registered Funders").register()}, - {"argos_grants", Gauge.build().name("argos_grants").help("Number of registered Grants").register()}, - {"argos_projects", Gauge.build().name("argos_projects").help("Number of registered Projects").register()}, - {"argos_researchers", Gauge.build().name("argos_researchers").help("Number of Colaborators/Researchers").register()}, - - {"argos_managed_dataset_descriptions", Gauge.build().name("argos_managed_dataset_descriptions").help("Number of managed Dataset Descriptions").labelNames("status").register()}, - - {"argos_dataset_templates", Gauge.build().name("argos_dataset_templates").help("Number of dataset Templates").labelNames("status").register()}, - - {"argos_users", Gauge.build().name("argos_users").help("Number of users").labelNames("type").register()}, - - {"argos_languages", Gauge.build().name("argos_languages").help("Number of Languages").register()}, - - }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1])); + private final Map gauges; - public static void increaseValue(String name, int amount, String label) { + public void increaseValue(String name, int amount, String label) { if(label != null) { gauges.get(name).labels(label).inc(amount); @@ -47,7 +37,7 @@ public class MetricsManager { } } - public static void decreaseValue(String name, int amount, String label) { + public void decreaseValue(String name, int amount, String label) { if(label != null) { gauges.get(name).labels(label).dec(amount); } else { @@ -55,7 +45,7 @@ public class MetricsManager { } } - public static Integer getValue(String name, String label) { + public Integer getValue(String name, String label) { if(label != null) { return Double.valueOf(gauges.get(name).labels(label).get()).intValue(); } else { @@ -63,7 +53,7 @@ public class MetricsManager { } } - public static void calculateValue(String name, int amount, String label) { + public void calculateValue(String name, int amount, String label) { Integer orig = getValue(name, label); int diff = orig - amount; if (diff != 0) { @@ -75,19 +65,34 @@ public class MetricsManager { } } - private final DatasetManager datasetManager; - private final DataManagementPlanManager dataManagementPlanManager; - private final DatasetProfileManager datasetProfileManager; + private final ApiContext apiContext; private final UserManager userManager; private final Environment environment; @Autowired - public MetricsManager(DatasetManager datasetManager, DataManagementPlanManager dataManagementPlanManager, DatasetProfileManager datasetProfileManager, UserManager userManager, Environment environment) { - this.datasetManager = datasetManager; - this.dataManagementPlanManager = dataManagementPlanManager; - this.datasetProfileManager = datasetProfileManager; + public MetricsManager(ApiContext apiContext, UserManager userManager, Environment environment, PrometheusMeterRegistry registry) { + this.apiContext = apiContext; this.userManager = userManager; this.environment = environment; + registry.clear(); + this.gauges = Stream.of( new Object[][]{ + {"argos_managed_dmps", Gauge.build().name("argos_managed_dmps").help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())}, + + {"argos_funders", Gauge.build().name("argos_funders").help("Number of registered Funders").register(registry.getPrometheusRegistry())}, + {"argos_grants", Gauge.build().name("argos_grants").help("Number of registered Grants").register(registry.getPrometheusRegistry())}, + {"argos_projects", Gauge.build().name("argos_projects").help("Number of registered Projects").register(registry.getPrometheusRegistry())}, + {"argos_researchers", Gauge.build().name("argos_researchers").help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())}, + + {"argos_managed_dataset_descriptions", Gauge.build().name("argos_managed_dataset_descriptions").help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())}, + + {"argos_dataset_templates", Gauge.build().name("argos_dataset_templates").help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())}, + + {"argos_users", Gauge.build().name("argos_users").help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())}, + + {"argos_languages", Gauge.build().name("argos_languages").help("Number of Languages").register(registry.getPrometheusRegistry())}, + + }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1])); + } @PostConstruct @@ -95,24 +100,24 @@ public class MetricsManager { @Scheduled(initialDelay = 1000 * 60 * 60, fixedDelay = 1000 * 60 * 60) public void init() throws IOException { logger.info("Start calculating Metrics"); - calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllDrafts(), "draft"); - calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllFinalized(), "finalized"); - calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllPublished(), "published"); - calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllDoied(), "doied"); + calculateValue("argos_managed_dmps", (int) countAllDraftDMPs(), "draft"); + calculateValue("argos_managed_dmps", (int) countAllFinalizedDMPs(), "finalized"); + calculateValue("argos_managed_dmps", (int) countAllPublishedDMPs(), "published"); + calculateValue("argos_managed_dmps", (int) countAllDoiedDMPs(), "doied"); - calculateValue("argos_funders", (int) dataManagementPlanManager.countAllFunders(), null); - calculateValue("argos_grants", (int) dataManagementPlanManager.countAllGrants(), null); - calculateValue("argos_projects", (int) dataManagementPlanManager.countAllProjects(), null); - calculateValue("argos_researchers", (int) dataManagementPlanManager.countAllResearchers(), null); + calculateValue("argos_funders", (int) countAllFunders(), null); + calculateValue("argos_grants", (int) countAllGrants(), null); + calculateValue("argos_projects", (int) countAllProjects(), null); + calculateValue("argos_researchers", (int) countAllResearchers(), null); - calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllDraft(), "draft"); - calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllFinalized(), "finalized"); - calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllPublic(), "published"); - calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllWithDoi(), "doied"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllDraftDatasets(), "draft"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllFinalizedDatasets(), "finalized"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllPublicDatasets(), "published"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllDatasetsWithDoi(), "doied"); - calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAlldraft(), "draft"); - calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAllFinalized(), "active"); - calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAllUsed(), "used"); + calculateValue("argos_dataset_templates", (int) countAllDraftTemplates(), "draft"); + calculateValue("argos_dataset_templates", (int) countAllFinalizedTemplates(), "active"); + calculateValue("argos_dataset_templates", (int) countAllUsedTemplates(), "used"); calculateValue("argos_users", (int) userManager.countActiveUsers().intValue(), "loggedin"); calculateValue("argos_users", (int) userManager.countAllUsers().intValue(), "total"); @@ -123,4 +128,103 @@ public class MetricsManager { logger.info("Metrics calculation Completed"); } + private long countAllDraftDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllFinalizedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllPublishedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setIsPublic(true); + criteria.setOnlyPublic(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllDoiedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setHasDoi(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllResearchers() { + ResearcherCriteria criteria = new ResearcherCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count(); + } + + private long countAllProjects() { + ProjectCriteria criteria = new ProjectCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count(); + } + + private long countAllFunders() { + FunderCriteria criteria = new FunderCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count(); + } + + private long countAllGrants() { + GrantCriteria criteria = new GrantCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count(); + } + + public long countAllDraftDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllFinalizedDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllPublicDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setIsPublic(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllDatasetsWithDoi() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setHasDoi(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllDraftTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); + } + + public long countAllFinalizedTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); + } + + @Transactional + public long countAllUsedTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(1); + criteria.setAllVersions(false); + List datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList(); + List filteredProfiles = new ArrayList<>(); + for (DatasetProfile datasetProfile : datasetProfiles) { + DatasetCriteria datasetCriteria = new DatasetCriteria(); + datasetCriteria.setDatasetTemplates(Collections.singletonList(datasetProfile.getId())); + Long datasetCount = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria).count(); + if (datasetCount > 0) { + filteredProfiles.add(datasetProfile); + } + } + return filteredProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).count(); + } + } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index 7b1bb2ac9..43c645f60 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -459,7 +459,7 @@ public class RemoteFetcher { try { String filePath = Paths.get(path).toUri().toURL().toString(); ObjectMapper mapper = new ObjectMapper(); - internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); + internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); return searchListMap(internalResults, query); } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/dmp-backend/web/src/main/resources/config/application.properties b/dmp-backend/web/src/main/resources/config/application.properties index 34fd880db..a12b86668 100644 --- a/dmp-backend/web/src/main/resources/config/application.properties +++ b/dmp-backend/web/src/main/resources/config/application.properties @@ -5,6 +5,19 @@ logging.file=/logs/spring-boot-logging.log spring.profiles.active=devel eu.eudat.logic.proxy.allowed.host=https://eestore.paas2.uninett.no +####################Metrics############## +management.endpoint.metrics.enabled=false +management.metrics.enable.http=false +management.metrics.enable.jvm=false +management.metrics.enable.jdbc=false +management.metrics.enable.tomcat=false +management.metrics.enable.logback=false +management.metrics.enable.hikaricp=false +management.metrics.enable.cache=false +management.endpoints.web.exposure.include=prometheus +management.endpoint.prometheus.enabled=true +management.metrics.export.prometheus.enabled=true + ####################INVITATION MAIL CONFIGURATIONS############## ####################GENERIC MAIL CONFIGURATIONS################# mail.subject=Invitation to DMP Plan {dmpname} From 6f92409f1e8831a3f7ad6f96f77849560eed0e12 Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Tue, 29 Jun 2021 14:33:57 +0300 Subject: [PATCH 2/4] Update Spring boot framework to version 2.5.2 --- dmp-backend/pom.xml | 64 +++--- dmp-backend/web/pom.xml | 31 +-- .../main/java/eu/eudat/EuDatApplication.java | 2 +- .../java/eu/eudat/cache/ResponsesCache.java | 30 +-- .../DevelDatabaseConfiguration.java | 2 +- .../configurations/WebMVCConfiguration.java | 2 - .../main/java/eu/eudat/controllers/Admin.java | 8 +- .../main/java/eu/eudat/controllers/Login.java | 11 +- .../controllers/UserGuideController.java | 6 +- .../managers/DataManagementPlanManager.java | 87 ++------ .../eudat/logic/managers/DatasetManager.java | 34 +--- .../logic/managers/DatasetProfileManager.java | 29 +-- .../eudat/logic/managers/MetricsManager.java | 192 ++++++++++++++---- .../logic/proxy/fetching/RemoteFetcher.java | 2 +- .../resources/config/application.properties | 13 ++ 15 files changed, 287 insertions(+), 226 deletions(-) diff --git a/dmp-backend/pom.xml b/dmp-backend/pom.xml index 254577ace..be51b3cd1 100644 --- a/dmp-backend/pom.xml +++ b/dmp-backend/pom.xml @@ -10,7 +10,7 @@ org.springframework.boot spring-boot-starter-parent - 1.5.9.RELEASE + 2.5.2 @@ -27,25 +27,26 @@ 1.8 0.0.1-SNAPSHOT - 4.3.8.RELEASE + 5.3.8 - 4.2.3.RELEASE - 1.19.1 + 5.3.10.RELEASE + + + - 5.2.11.Final + 5.5.3.Final 1.9 4.11 1.2.17 1.7.12 - 9.0.7.v20131107 - + 1.2.3 - 1 - 3.0.1 - ikalyvas + + @@ -58,7 +59,7 @@ org.springframework spring-context-support - 4.0.0.RELEASE + ${org.springframework.version} org.springframework.boot @@ -139,7 +140,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.9.3 + 2.12.3 @@ -147,13 +148,13 @@ com.fasterxml.jackson.core jackson-core - 2.9.4 + 2.12.3 com.fasterxml.jackson.core jackson-databind - 2.9.4 + 2.12.3 @@ -257,36 +258,53 @@ 1.3.1 + + + javax.validation + validation-api + 2.0.1.Final + + + io.prometheus simpleclient - 0.10.0 + 0.11.0 io.prometheus simpleclient_hotspot - 0.10.0 + 0.11.0 io.prometheus simpleclient_httpserver - 0.10.0 + 0.11.0 io.prometheus simpleclient_pushgateway - 0.10.0 + 0.11.0 - + - io.prometheus - simpleclient_spring_boot - 0.10.0 + io.micrometer + micrometer-registry-prometheus + 1.7.1 + + + org.springframework.boot + spring-boot-starter-actuator + + + + io.micrometer + micrometer-core diff --git a/dmp-backend/web/pom.xml b/dmp-backend/web/pom.xml index d3cfd31e6..6798b6022 100644 --- a/dmp-backend/web/pom.xml +++ b/dmp-backend/web/pom.xml @@ -50,31 +50,37 @@ com.fasterxml.jackson.core jackson-annotations - 2.9.0 + 2.12.3 - + - + + + com.github.ben-manes.caffeine + caffeine + 2.9.0 + + - junit - junit - 4.12 + org.junit.jupiter + junit-jupiter-api + 5.7.2 test @@ -87,7 +93,7 @@ io.springfox springfox-swagger-ui - 2.7.0 + 3.0.0 @@ -101,7 +107,7 @@ org.springframework.social spring-social-linkedin - 1.0.0.RELEASE + 1.0.2.RELEASE @@ -143,7 +149,6 @@ org.springframework.boot spring-boot-maven-plugin - 1.5.9.RELEASE diff --git a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java index d9c2b65d6..31e87ddf5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java +++ b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java @@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.web.support.SpringBootServletInitializer; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; import org.springframework.scheduling.annotation.EnableAsync; @SpringBootApplication diff --git a/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java b/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java index 0a09a6731..427b61173 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java +++ b/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java @@ -1,12 +1,12 @@ package eu.eudat.cache; -import com.google.common.cache.CacheBuilder; +import com.github.benmanes.caffeine.cache.Caffeine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; -import org.springframework.cache.guava.GuavaCache; +import org.springframework.cache.caffeine.CaffeineCache; import org.springframework.cache.support.SimpleCacheManager; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; @@ -29,19 +29,19 @@ public class ResponsesCache { public CacheManager cacheManager() { logger.info("Loading ResponsesCache..."); SimpleCacheManager simpleCacheManager = new SimpleCacheManager(); - List caches = new ArrayList(); - caches.add(new GuavaCache("repositories", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("grants", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("projects", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("funders", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("externalDatasets", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("currencies", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("licenses", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + List caches = new ArrayList(); + caches.add(new CaffeineCache("repositories", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("grants", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("projects", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("funders", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("organisations", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("registries", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("services", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("tags", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("researchers", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("externalDatasets", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("currencies", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("licenses", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); simpleCacheManager.setCaches(caches); logger.info("OK"); return simpleCacheManager; diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java index 1c5c48bed..a1311f1f1 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java @@ -2,8 +2,8 @@ package eu.eudat.configurations; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; import org.springframework.context.annotation.*; import org.springframework.core.env.Environment; import org.springframework.dao.annotation.PersistenceExceptionTranslationPostProcessor; diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java index 2c64aa8d1..36d9c502c 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java @@ -4,7 +4,6 @@ import eu.eudat.controllers.interceptors.RequestInterceptor; import eu.eudat.logic.handlers.PrincipalArgumentResolver; import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.services.operations.authentication.AuthenticationService; -import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableAsync; @@ -18,7 +17,6 @@ import java.util.List; @EnableAsync @Configuration @EnableScheduling -@EnablePrometheusEndpoint public class WebMVCConfiguration extends WebMvcConfigurerAdapter { private ApiContext apiContext; diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java index 6a89b4429..8cee99ae8 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java @@ -44,13 +44,15 @@ public class Admin extends BaseController { private DatasetProfileManager datasetProfileManager; private UserManager userManager; private ConfigLoader configLoader; + private final MetricsManager metricsManager; @Autowired - public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader) { + public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader, MetricsManager metricsManager) { super(apiContext); this.datasetProfileManager = datasetProfileManager; this.userManager = userManager; this.configLoader = configLoader; + this.metricsManager = metricsManager; } @Transactional @@ -70,7 +72,7 @@ public class Admin extends BaseController { userDatasetProfile.setRole(0); getApiContext().getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile); datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile); - MetricsManager.increaseValue("argos_dataset_templates", 1, "total"); + metricsManager.increaseValue("argos_dataset_templates", 1, "total"); return ResponseEntity.status(HttpStatus.OK).body(modelDefinition.getId()); } @@ -89,7 +91,7 @@ public class Admin extends BaseController { eu.eudat.data.entities.DatasetProfile datasetProfile = this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(datasetprofile); datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile); if (datasetProfile.getStatus() == 1 && oldStatus == 0) { - MetricsManager.increaseValue("argos_dataset_templates", 1, "active"); + metricsManager.increaseValue("argos_dataset_templates", 1, "active"); } return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().status(ApiMessageCode.NO_MESSAGE)); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java index ac75578d9..a6172f284 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java @@ -64,6 +64,7 @@ public class Login { private ConfigurableProviderTokenValidator configurableProviderTokenValidator; private ZenodoTokenValidator zenodoTokenValidator; private ConfigLoader configLoader; + private final MetricsManager metricsManager; // private Logger logger; @@ -75,7 +76,8 @@ public class Login { B2AccessTokenValidator b2AccessTokenValidator, ORCIDTokenValidator orcidTokenValidator, LinkedInTokenValidator linkedInTokenValidator, OpenAIRETokenValidator openAIRETokenValidator, ConfigurableProviderTokenValidator configurableProviderTokenValidator, ZenodoTokenValidator zenodoTokenValidator, - ConfigLoader configLoader, UserManager userManager) { + ConfigLoader configLoader, UserManager userManager, + MetricsManager metricsManager) { this.customAuthenticationProvider = customAuthenticationProvider; this.nonVerifiedUserAuthenticationService = nonVerifiedUserAuthenticationService; this.twitterTokenValidator = twitterTokenValidator; @@ -87,6 +89,7 @@ public class Login { this.zenodoTokenValidator = zenodoTokenValidator; this.configLoader = configLoader; this.userManager = userManager; + this.metricsManager = metricsManager; } @@ -95,7 +98,7 @@ public class Login { public @ResponseBody ResponseEntity> externallogin(@RequestBody LoginInfo credentials) throws GeneralSecurityException, NullEmailException { logger.info("Trying To Login With " + credentials.getProvider()); - MetricsManager.increaseValue("argos_users", 1, "loggedin"); + metricsManager.increaseValue("argos_users", 1, "loggedin"); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(customAuthenticationProvider.authenticate(credentials)).status(ApiMessageCode.SUCCESS_MESSAGE)); } @@ -104,7 +107,7 @@ public class Login { public @ResponseBody ResponseEntity> nativelogin(@RequestBody Credentials credentials) throws NullEmailException { logger.info(credentials.getUsername() + " Trying To Login"); - MetricsManager.increaseValue("argos_users", 1, "loggedin"); + metricsManager.increaseValue("argos_users", 1, "loggedin"); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(userManager.authenticate(this.nonVerifiedUserAuthenticationService, credentials)).status(ApiMessageCode.SUCCESS_MESSAGE)); } @@ -165,7 +168,7 @@ public class Login { ResponseEntity> logout(Principal principal) { this.nonVerifiedUserAuthenticationService.Logout(principal.getToken()); logger.info(principal + " Logged Out"); - MetricsManager.decreaseValue("argos_users", 1, "loggedin"); + metricsManager.decreaseValue("argos_users", 1, "loggedin"); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().status(ApiMessageCode.NO_MESSAGE)); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java index b7b07e474..c3aa00a44 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java @@ -30,16 +30,18 @@ import static eu.eudat.types.Authorities.ADMIN; public class UserGuideController { private Environment environment; + private final MetricsManager metricsManager; @Autowired - public UserGuideController(Environment environment) { + public UserGuideController(Environment environment, MetricsManager metricsManager) { this.environment = environment; + this.metricsManager = metricsManager; } @RequestMapping(path = "{lang}", method = RequestMethod.GET ) public ResponseEntity getUserGuide(@PathVariable(name = "lang") String lang) throws IOException { long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count(); - MetricsManager.calculateValue("argos_languages", (int) files, null); + metricsManager.calculateValue("argos_languages", (int) files, null); Stream walk = Files.walk(Paths.get(this.environment.getProperty("userguide.path"))); List result = walk.filter(Files::isRegularFile) .map(Path::toString).collect(Collectors.toList()); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java index 5a070671b..563bb10ce 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java @@ -108,9 +108,10 @@ public class DataManagementPlanManager { private Environment environment; private RDAManager rdaManager; private UserManager userManager; + private final MetricsManager metricsManager; @Autowired - public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager) { + public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager, MetricsManager metricsManager) { this.apiContext = apiContext; this.datasetManager = datasetManager; this.utilitiesService = apiContext.getUtilitiesService(); @@ -118,6 +119,7 @@ public class DataManagementPlanManager { this.environment = environment; this.rdaManager = rdaManager; this.userManager = userManager; + this.metricsManager = metricsManager; } /* @@ -441,7 +443,7 @@ public class DataManagementPlanManager { setNotification = true; } else { - MetricsManager.increaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.increaseValue("argos_managed_dmps", 1, "draft"); } DMP newDmp = dataManagementPlan.toDataModel(); @@ -565,7 +567,7 @@ public class DataManagementPlanManager { if (dataManagementPlan.getStatus() == (int) DMP.DMPStatus.FINALISED.getValue() && dmp1.getStatus().equals(DMP.DMPStatus.FINALISED.getValue())) throw new Exception("DMP is finalized, therefore cannot be edited."); } else { - MetricsManager.increaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.increaseValue("argos_managed_dmps", 1, "draft"); } List datasets = new ArrayList<>(); DMP tempDMP = dataManagementPlan.toDataModel(); @@ -691,7 +693,7 @@ public class DataManagementPlanManager { newDmp.setDataset(new HashSet<>(databaseRepository.getDatasetDao().getWithCriteria(criteria1).toList())); this.updateIndex(newDmp); - MetricsManager.increaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.increaseValue("argos_managed_dmps", 1, "draft"); return newDmp.getId(); } @@ -705,16 +707,16 @@ public class DataManagementPlanManager { DMP oldDmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(uuid); switch (oldDmp.getStatus()) { case 0: - MetricsManager.decreaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "draft"); break; case 1: if (oldDmp.getDoi() != null) { - MetricsManager.decreaseValue("argos_managed_dmps", 1, "doied"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "doied"); } if (oldDmp.isPublic()) { - MetricsManager.decreaseValue("argos_managed_dmps", 1, "published"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "published"); } - MetricsManager.decreaseValue("argos_managed_dmps", 1, "finalized"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "finalized"); break; } oldDmp.setStatus(DMP.DMPStatus.DELETED.getValue()); @@ -754,7 +756,7 @@ public class DataManagementPlanManager { else { researcher.setCreationUser(user); researcherRepository.createOrUpdate(researcher); - MetricsManager.increaseValue("argos_researchers", 1, null); + metricsManager.increaseValue("argos_researchers", 1, null); } } } @@ -836,14 +838,14 @@ public class DataManagementPlanManager { project.setType(Project.ProjectType.EXTERNAL.getValue()); if (project.getId() == null) project.setId(UUID.randomUUID()); projectDao.createOrUpdate(project); - MetricsManager.increaseValue("argos_projects", 1, null); + metricsManager.increaseValue("argos_projects", 1, null); } } else { project.setType(Project.ProjectType.EXTERNAL.getValue()); if (project.getId() == null) project.setId(UUID.randomUUID()); projectDao.createOrUpdate(project); - MetricsManager.increaseValue("argos_projects", 1, null); + metricsManager.increaseValue("argos_projects", 1, null); } } } @@ -920,7 +922,7 @@ public class DataManagementPlanManager { return newDataset; }).thenApplyAsync(item -> { futures.add(datasetDao.createOrUpdateAsync(item).whenComplete(((dataset1, throwable) -> { - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); eu.eudat.elastic.entities.Dataset datasetElastic = new eu.eudat.elastic.entities.Dataset(); datasetElastic.setId(dataset1.getId().toString()); datasetElastic.setLabel(dataset1.getLabel()); @@ -979,7 +981,7 @@ public class DataManagementPlanManager { dmp.setPublic(true); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp); this.updateIndex(dmp); - MetricsManager.increaseValue("argos_managed_dmps", 1, "published"); + metricsManager.increaseValue("argos_managed_dmps", 1, "published"); DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setGroupIds(Collections.singletonList(dmp.getGroupId())); criteria.setAllVersions(true); @@ -992,7 +994,7 @@ public class DataManagementPlanManager { tags = elastic.getTags(); } this.datasetManager.updateTags(dataset, tags); - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published"); } catch (Exception e) { logger.error(e.getMessage(), e); } @@ -1078,11 +1080,11 @@ public class DataManagementPlanManager { this.updateIndex(dmp); UserInfo user = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal.getId()); sendNotification(dmp, user, NotificationType.DMP_FINALISED); - MetricsManager.decreaseValue("argos_managed_dmps", 1, "draft"); - MetricsManager.increaseValue("argos_managed_dmps", 1, "finalized"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.increaseValue("argos_managed_dmps", 1, "finalized"); this.updateDatasetsIndex(indexDatasets); - MetricsManager.decreaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "draft"); - MetricsManager.increaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "finalized"); + metricsManager.decreaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "draft"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", indexDatasets.size(), "finalized"); } public void undoFinalize(UUID id, Principal principal) throws Exception { @@ -1094,8 +1096,8 @@ public class DataManagementPlanManager { dmp.setStatus(DMP.DMPStatus.ACTIVE.getValue()); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp); this.updateIndex(dmp); - MetricsManager.decreaseValue("argos_managed_dmps", 1, "finalized"); - MetricsManager.increaseValue("argos_managed_dmps", 1, "draft"); + metricsManager.decreaseValue("argos_managed_dmps", 1, "finalized"); + metricsManager.increaseValue("argos_managed_dmps", 1, "draft"); } public void updateUsers(UUID id, List users, Principal principal) throws Exception { @@ -2209,51 +2211,6 @@ public class DataManagementPlanManager { } - public long countAllDrafts() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllPublished() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setIsPublic(true); - criteria.setOnlyPublic(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllDoied() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setHasDoi(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllResearchers() { - ResearcherCriteria criteria = new ResearcherCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count(); - } - - public long countAllProjects() { - ProjectCriteria criteria = new ProjectCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count(); - } - - public long countAllFunders() { - FunderCriteria criteria = new FunderCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count(); - } - - public long countAllGrants() { - GrantCriteria criteria = new GrantCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count(); - } - /*public DataTableData getPublicPaged(DataManagmentPlanPublicTableRequest dataManagementPlanPublicTableRequest, String fieldsGroup, Principal principal) throws Exception { dataManagementPlanPublicTableRequest.setQuery(databaseRepository.getDmpDao().asQueryable().withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class))); QueryableList items = dataManagementPlanPublicTableRequest.applyCriteria(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java index b96934677..9005e631e 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java @@ -109,9 +109,10 @@ public class DatasetManager { private UserManager userManager; private ConfigLoader configLoader; private Environment environment; + private final MetricsManager metricsManager; @Autowired - public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment) { + public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment, MetricsManager metricsManager) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository(); @@ -119,6 +120,7 @@ public class DatasetManager { this.userManager = userManager; this.configLoader = configLoader; this.environment = environment; + this.metricsManager = metricsManager; } public DataTableData getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception { @@ -619,7 +621,7 @@ public class DatasetManager { sendNotification = true; } } else { - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); } if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null) throw new Exception("DMP is finalized, therefore Dataset cannot be edited."); @@ -889,7 +891,7 @@ public class DatasetManager { if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue()) throw new Exception("You cannot make public a Dataset That Has not Been Finalised"); datasetDao.createOrUpdate(dataset); - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "published"); } public ResponseEntity getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException { @@ -978,7 +980,7 @@ public class DatasetManager { createServicesIfTheyDontExist(entity); createExternalDatasetsIfTheyDontExist(entity); - MetricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); + metricsManager.increaseValue("argos_managed_dataset_descriptions", 1, "draft"); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity); } @@ -1136,28 +1138,4 @@ public class DatasetManager { listingModel.setProfileLatestVersion(islast); return listingModel; } - - public long countAllDraft() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllPublic() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setIsPublic(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllWithDoi() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setHasDoi(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java index 244287173..480bf3895 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java @@ -59,12 +59,14 @@ public class DatasetProfileManager { private DatabaseRepository databaseRepository; private Environment environment; private List cache; + private final MetricsManager metricsManager; @Autowired - public DatasetProfileManager(ApiContext apiContext, Environment environment) { + public DatasetProfileManager(ApiContext apiContext, Environment environment, MetricsManager metricsManager) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.environment = environment; + this.metricsManager = metricsManager; this.cache = new ArrayList<>(); } @@ -92,7 +94,7 @@ public class DatasetProfileManager { DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(UUID.fromString(id)); apiContext.getOperationsContext().getDatabaseRepository().detachEntity(profile); profile.setId(null); - MetricsManager.increaseValue("argos_dataset_templates", 1, "total"); + metricsManager.increaseValue("argos_dataset_templates", 1, "total"); return profile; } @@ -233,7 +235,7 @@ public class DatasetProfileManager { File localFile = convert(multiPartFile); eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileModel.DatasetProfile profile = xmlBuilder.build(localFile); Files.deleteIfExists(localFile.toPath()); - MetricsManager.increaseValue("argos_dataset_templates", 1, "total"); + metricsManager.increaseValue("argos_dataset_templates", 1, "total"); return profile; } catch (IOException e) { logger.error(e.getMessage(), e); @@ -316,25 +318,4 @@ public class DatasetProfileManager { }).collect(Collectors.toList())); } } - - public long countAlldraft() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); - } - - @Transactional - public long countAllUsed() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(1); - criteria.setAllVersions(false); - List datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList(); - return datasetProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).filter(datasetProfile -> !datasetProfile.getDataset().isEmpty()).count(); - } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java index 353171093..9dc432068 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java @@ -1,5 +1,11 @@ package eu.eudat.logic.managers; +import eu.eudat.data.dao.criteria.*; +import eu.eudat.data.entities.Dataset; +import eu.eudat.data.entities.DatasetProfile; +import eu.eudat.logic.services.ApiContext; +import eu.eudat.logic.utilities.helpers.StreamDistinctBy; +import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.Gauge; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -13,32 +19,16 @@ import javax.transaction.Transactional; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @Component public class MetricsManager { private final static Logger logger = LoggerFactory.getLogger(MetricsManager.class); - private static Map gauges = Stream.of( new Object[][]{ - {"argos_managed_dmps", Gauge.build().name("argos_managed_dmps").help("Number of managed DMPs").labelNames("status").register()}, - - {"argos_funders", Gauge.build().name("argos_funders").help("Number of registered Funders").register()}, - {"argos_grants", Gauge.build().name("argos_grants").help("Number of registered Grants").register()}, - {"argos_projects", Gauge.build().name("argos_projects").help("Number of registered Projects").register()}, - {"argos_researchers", Gauge.build().name("argos_researchers").help("Number of Colaborators/Researchers").register()}, - - {"argos_managed_dataset_descriptions", Gauge.build().name("argos_managed_dataset_descriptions").help("Number of managed Dataset Descriptions").labelNames("status").register()}, - - {"argos_dataset_templates", Gauge.build().name("argos_dataset_templates").help("Number of dataset Templates").labelNames("status").register()}, - - {"argos_users", Gauge.build().name("argos_users").help("Number of users").labelNames("type").register()}, - - {"argos_languages", Gauge.build().name("argos_languages").help("Number of Languages").register()}, - - }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1])); + private final Map gauges; - public static void increaseValue(String name, int amount, String label) { + public void increaseValue(String name, int amount, String label) { if(label != null) { gauges.get(name).labels(label).inc(amount); @@ -47,7 +37,7 @@ public class MetricsManager { } } - public static void decreaseValue(String name, int amount, String label) { + public void decreaseValue(String name, int amount, String label) { if(label != null) { gauges.get(name).labels(label).dec(amount); } else { @@ -55,7 +45,7 @@ public class MetricsManager { } } - public static Integer getValue(String name, String label) { + public Integer getValue(String name, String label) { if(label != null) { return Double.valueOf(gauges.get(name).labels(label).get()).intValue(); } else { @@ -63,7 +53,7 @@ public class MetricsManager { } } - public static void calculateValue(String name, int amount, String label) { + public void calculateValue(String name, int amount, String label) { Integer orig = getValue(name, label); int diff = orig - amount; if (diff != 0) { @@ -75,19 +65,34 @@ public class MetricsManager { } } - private final DatasetManager datasetManager; - private final DataManagementPlanManager dataManagementPlanManager; - private final DatasetProfileManager datasetProfileManager; + private final ApiContext apiContext; private final UserManager userManager; private final Environment environment; @Autowired - public MetricsManager(DatasetManager datasetManager, DataManagementPlanManager dataManagementPlanManager, DatasetProfileManager datasetProfileManager, UserManager userManager, Environment environment) { - this.datasetManager = datasetManager; - this.dataManagementPlanManager = dataManagementPlanManager; - this.datasetProfileManager = datasetProfileManager; + public MetricsManager(ApiContext apiContext, UserManager userManager, Environment environment, PrometheusMeterRegistry registry) { + this.apiContext = apiContext; this.userManager = userManager; this.environment = environment; + registry.clear(); + this.gauges = Stream.of( new Object[][]{ + {"argos_managed_dmps", Gauge.build().name("argos_managed_dmps").help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())}, + + {"argos_funders", Gauge.build().name("argos_funders").help("Number of registered Funders").register(registry.getPrometheusRegistry())}, + {"argos_grants", Gauge.build().name("argos_grants").help("Number of registered Grants").register(registry.getPrometheusRegistry())}, + {"argos_projects", Gauge.build().name("argos_projects").help("Number of registered Projects").register(registry.getPrometheusRegistry())}, + {"argos_researchers", Gauge.build().name("argos_researchers").help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())}, + + {"argos_managed_dataset_descriptions", Gauge.build().name("argos_managed_dataset_descriptions").help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())}, + + {"argos_dataset_templates", Gauge.build().name("argos_dataset_templates").help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())}, + + {"argos_users", Gauge.build().name("argos_users").help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())}, + + {"argos_languages", Gauge.build().name("argos_languages").help("Number of Languages").register(registry.getPrometheusRegistry())}, + + }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1])); + } @PostConstruct @@ -95,24 +100,24 @@ public class MetricsManager { @Scheduled(initialDelay = 1000 * 60 * 60, fixedDelay = 1000 * 60 * 60) public void init() throws IOException { logger.info("Start calculating Metrics"); - calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllDrafts(), "draft"); - calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllFinalized(), "finalized"); - calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllPublished(), "published"); - calculateValue("argos_managed_dmps", (int) dataManagementPlanManager.countAllDoied(), "doied"); + calculateValue("argos_managed_dmps", (int) countAllDraftDMPs(), "draft"); + calculateValue("argos_managed_dmps", (int) countAllFinalizedDMPs(), "finalized"); + calculateValue("argos_managed_dmps", (int) countAllPublishedDMPs(), "published"); + calculateValue("argos_managed_dmps", (int) countAllDoiedDMPs(), "doied"); - calculateValue("argos_funders", (int) dataManagementPlanManager.countAllFunders(), null); - calculateValue("argos_grants", (int) dataManagementPlanManager.countAllGrants(), null); - calculateValue("argos_projects", (int) dataManagementPlanManager.countAllProjects(), null); - calculateValue("argos_researchers", (int) dataManagementPlanManager.countAllResearchers(), null); + calculateValue("argos_funders", (int) countAllFunders(), null); + calculateValue("argos_grants", (int) countAllGrants(), null); + calculateValue("argos_projects", (int) countAllProjects(), null); + calculateValue("argos_researchers", (int) countAllResearchers(), null); - calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllDraft(), "draft"); - calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllFinalized(), "finalized"); - calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllPublic(), "published"); - calculateValue("argos_managed_dataset_descriptions", (int) datasetManager.countAllWithDoi(), "doied"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllDraftDatasets(), "draft"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllFinalizedDatasets(), "finalized"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllPublicDatasets(), "published"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllDatasetsWithDoi(), "doied"); - calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAlldraft(), "draft"); - calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAllFinalized(), "active"); - calculateValue("argos_dataset_templates", (int) datasetProfileManager.countAllUsed(), "used"); + calculateValue("argos_dataset_templates", (int) countAllDraftTemplates(), "draft"); + calculateValue("argos_dataset_templates", (int) countAllFinalizedTemplates(), "active"); + calculateValue("argos_dataset_templates", (int) countAllUsedTemplates(), "used"); calculateValue("argos_users", (int) userManager.countActiveUsers().intValue(), "loggedin"); calculateValue("argos_users", (int) userManager.countAllUsers().intValue(), "total"); @@ -123,4 +128,103 @@ public class MetricsManager { logger.info("Metrics calculation Completed"); } + private long countAllDraftDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllFinalizedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllPublishedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setIsPublic(true); + criteria.setOnlyPublic(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllDoiedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setHasDoi(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllResearchers() { + ResearcherCriteria criteria = new ResearcherCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count(); + } + + private long countAllProjects() { + ProjectCriteria criteria = new ProjectCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count(); + } + + private long countAllFunders() { + FunderCriteria criteria = new FunderCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count(); + } + + private long countAllGrants() { + GrantCriteria criteria = new GrantCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count(); + } + + public long countAllDraftDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllFinalizedDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllPublicDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setIsPublic(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllDatasetsWithDoi() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setHasDoi(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllDraftTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); + } + + public long countAllFinalizedTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); + } + + @Transactional + public long countAllUsedTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(1); + criteria.setAllVersions(false); + List datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList(); + List filteredProfiles = new ArrayList<>(); + for (DatasetProfile datasetProfile : datasetProfiles) { + DatasetCriteria datasetCriteria = new DatasetCriteria(); + datasetCriteria.setDatasetTemplates(Collections.singletonList(datasetProfile.getId())); + Long datasetCount = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria).count(); + if (datasetCount > 0) { + filteredProfiles.add(datasetProfile); + } + } + return filteredProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).count(); + } + } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index 7b1bb2ac9..43c645f60 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -459,7 +459,7 @@ public class RemoteFetcher { try { String filePath = Paths.get(path).toUri().toURL().toString(); ObjectMapper mapper = new ObjectMapper(); - internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); + internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); return searchListMap(internalResults, query); } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/dmp-backend/web/src/main/resources/config/application.properties b/dmp-backend/web/src/main/resources/config/application.properties index 34fd880db..a12b86668 100644 --- a/dmp-backend/web/src/main/resources/config/application.properties +++ b/dmp-backend/web/src/main/resources/config/application.properties @@ -5,6 +5,19 @@ logging.file=/logs/spring-boot-logging.log spring.profiles.active=devel eu.eudat.logic.proxy.allowed.host=https://eestore.paas2.uninett.no +####################Metrics############## +management.endpoint.metrics.enabled=false +management.metrics.enable.http=false +management.metrics.enable.jvm=false +management.metrics.enable.jdbc=false +management.metrics.enable.tomcat=false +management.metrics.enable.logback=false +management.metrics.enable.hikaricp=false +management.metrics.enable.cache=false +management.endpoints.web.exposure.include=prometheus +management.endpoint.prometheus.enabled=true +management.metrics.export.prometheus.enabled=true + ####################INVITATION MAIL CONFIGURATIONS############## ####################GENERIC MAIL CONFIGURATIONS################# mail.subject=Invitation to DMP Plan {dmpname} From 0257ff713bf10dcddf009486a7b3401cd109f62e Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Tue, 7 Sep 2021 10:29:57 +0300 Subject: [PATCH 3/4] Remove the actuator from the prometheus path --- dmp-backend/web/src/main/resources/config/application.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/dmp-backend/web/src/main/resources/config/application.properties b/dmp-backend/web/src/main/resources/config/application.properties index a12b86668..1103dca16 100644 --- a/dmp-backend/web/src/main/resources/config/application.properties +++ b/dmp-backend/web/src/main/resources/config/application.properties @@ -14,6 +14,7 @@ management.metrics.enable.tomcat=false management.metrics.enable.logback=false management.metrics.enable.hikaricp=false management.metrics.enable.cache=false +management.endpoints.web.base-path=/ management.endpoints.web.exposure.include=prometheus management.endpoint.prometheus.enabled=true management.metrics.export.prometheus.enabled=true From 1ae278845309257d80e8d8cba2f659ddb60e2eeb Mon Sep 17 00:00:00 2001 From: George Kalampokis Date: Tue, 14 Sep 2021 17:43:02 +0300 Subject: [PATCH 4/4] Fix various issues with spring update --- .../java/eu/eudat/data/entities/Grant.java | 4 +- .../managers/DataManagementPlanManager.java | 40 ++++++++++++++++--- .../eudat/logic/managers/DatasetManager.java | 2 +- .../DataManagementPlanNewVersionModel.java | 15 ++++--- 4 files changed, 48 insertions(+), 13 deletions(-) diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java b/dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java index 9809f0953..fd5c77164 100644 --- a/dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java +++ b/dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java @@ -106,11 +106,11 @@ public class Grant implements DataEntity { @Column(name = "\"Definition\"", columnDefinition = "xml", nullable = true) private String definition; - @Column(name = "\"StartDate\"", nullable = false) + @Column(name = "\"StartDate\"", nullable = true) @Convert(converter = DateToUTCConverter.class) private Date startdate = null; - @Column(name = "\"EndDate\"", nullable = false) + @Column(name = "\"EndDate\"", nullable = true) @Convert(converter = DateToUTCConverter.class) private Date enddate = null; diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java index 563bb10ce..40f395864 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java @@ -151,8 +151,11 @@ public class DataManagementPlanManager { items = null; } } + if (items == null) { items = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dataManagementPlanTableRequest.getCriteria()); + } + List roles = new LinkedList<>(); if (!dataManagementPlanTableRequest.getCriteria().isOnlyPublic()) { if (dataManagementPlanTableRequest.getCriteria().getRole() != null) @@ -161,8 +164,11 @@ public class DataManagementPlanManager { } else { authItems = items; } + if (items == null) { totalData = authItems.count(); items = PaginationManager.applyPaging(authItems, dataManagementPlanTableRequest); + } else { + items = authItems; } @@ -534,6 +540,10 @@ public class DataManagementPlanManager { if (dataManagementPlan.getAssociatedUsers().size() == 0) assignUser(newDmp, user); + UUID dmpId = newDmp.getId(); + Set users = new HashSet<>(databaseRepository.getUserDmpDao().asQueryable().where((builder, root) -> builder.equal(root.get("dmp").get("id"), dmpId)).toList()); + newDmp.setUsers(users); + this.updateIndex(newDmp); if (setNotification) { @@ -625,13 +635,16 @@ public class DataManagementPlanManager { assignGrandUserIfInternal(newDmp, user); assignFunderUserIfInternal(newDmp, user); assignProjectUserIfInternal(newDmp, user); + if (newDmp.getGrant().getStartdate() == null) { + newDmp.getGrant().setStartdate(new Date()); + } + if (newDmp.getGrant().getEnddate() == null) { + newDmp.getGrant().setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS))); + } databaseRepository.getGrantDao().createOrUpdate(newDmp.getGrant()); DMP tempDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp); newDmp.setId(tempDmp.getId()); - // Assign creator. - assignUser(newDmp, user); - copyDatasets(newDmp, databaseRepository.getDatasetDao()); databaseRepository @@ -797,6 +810,12 @@ public class DataManagementPlanManager { else { grant.setType(Grant.GrantType.EXTERNAL.getValue()); grant.setCreationUser(null); + if (grant.getStartdate() == null) { + grant.setStartdate(new Date()); + } + if (grant.getEnddate() == null) { + grant.setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS))); + } grantDao.createOrUpdate(grant); } } @@ -1749,7 +1768,16 @@ public class DataManagementPlanManager { userDMP.setDmp(dmp); userDMP.setUser(userInfo); userDMP.setRole(role.getValue()); - databaseRepository.getUserDmpDao().createOrUpdate(userDMP); + userDMP = databaseRepository.getUserDmpDao().createOrUpdate(userDMP); + logger.debug("UserDMP Id: " + userDMP.getId()); + } + + private boolean userExists(UserDMP userDMP, UserInfo userInfo, UserDMP.UserDMPRoles role) { + UserDMP tempUser = userDMP; + if (userDMP.getUser() == null) { + tempUser = databaseRepository.getUserDmpDao().find(userDMP.getId()); + } + return tempUser.getUser().getId().equals(userInfo.getId()) && tempUser.getRole().equals(role.getValue()); } private void clearUsers(DMP dmp) { @@ -2147,7 +2175,9 @@ public class DataManagementPlanManager { jsonFos.flush(); } fileSystemResource = new FileSystemResource(tempJsonFile); - addFileMapRequest = new HttpEntity<>(fileSystemResource, null); + HttpHeaders jsonHeaders = new HttpHeaders(); + jsonHeaders.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); + addFileMapRequest = new HttpEntity<>(fileSystemResource, jsonHeaders); String jsonFileName = jsonFile.getHeaders().get("Content-Disposition").get(0).substring(jsonFile.getHeaders().get("Content-Disposition").get(0).lastIndexOf('=') + 1); addFileUrl = links.get("bucket") + "/" + jsonFileName + "?access_token=" + zenodoToken; restTemplate.put(addFileUrl, addFileMapRequest); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java index 9005e631e..23743dd9b 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java @@ -1095,7 +1095,7 @@ public class DatasetManager { if (!tagNodes.isEmpty()) { tagNodes.forEach(node -> { JsonNode value = node.get("value"); - String stringValue = value.asText().replaceAll("=", ":"); + String stringValue = value.toString().replaceAll("=", ":"); JSONArray values = new JSONArray(stringValue); if (values != null) { values.iterator().forEachRemaining(element -> { diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java index 46c193154..eae784aeb 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java @@ -7,7 +7,10 @@ import eu.eudat.models.data.funder.FunderDMPEditorModel; import eu.eudat.models.data.grant.GrantDMPEditorModel; import eu.eudat.models.data.project.ProjectDMPEditorModel; import eu.eudat.models.data.userinfo.UserInfo; +import eu.eudat.models.data.userinfo.UserListingModel; import net.minidev.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.*; import java.util.stream.Collectors; @@ -16,6 +19,8 @@ import java.util.stream.Collectors; * Created by ikalyvas on 2/5/2018. */ public class DataManagementPlanNewVersionModel implements DataModel { + private static final Logger logger = LoggerFactory.getLogger(DataManagementPlanNewVersionModel.class); + private UUID id; private String label; private UUID groupId; @@ -26,7 +31,7 @@ public class DataManagementPlanNewVersionModel implements DataModel organisations; private List researchers; - private List associatedUsers; + private List associatedUsers; private eu.eudat.models.data.userinfo.UserInfo creator; private Date created; private List datasets; @@ -104,10 +109,10 @@ public class DataManagementPlanNewVersionModel implements DataModel getAssociatedUsers() { + public List getAssociatedUsers() { return associatedUsers; } - public void setAssociatedUsers(List associatedUsers) { + public void setAssociatedUsers(List associatedUsers) { this.associatedUsers = associatedUsers; } @@ -163,7 +168,7 @@ public class DataManagementPlanNewVersionModel implements DataModel(new ArrayList<>(this.associatedUsers))); + // entity.setUsers(new HashSet<>(new ArrayList<>(this.associatedUsers))); entity.setDescription(this.description); entity.setStatus((short) this.status); entity.setGroupId(this.groupId == null ? UUID.randomUUID() : this.groupId); @@ -227,7 +232,7 @@ public class DataManagementPlanNewVersionModel implements DataModel