diff --git a/dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java b/dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java index 9809f0953..fd5c77164 100644 --- a/dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java +++ b/dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java @@ -106,11 +106,11 @@ public class Grant implements DataEntity { @Column(name = "\"Definition\"", columnDefinition = "xml", nullable = true) private String definition; - @Column(name = "\"StartDate\"", nullable = false) + @Column(name = "\"StartDate\"", nullable = true) @Convert(converter = DateToUTCConverter.class) private Date startdate = null; - @Column(name = "\"EndDate\"", nullable = false) + @Column(name = "\"EndDate\"", nullable = true) @Convert(converter = DateToUTCConverter.class) private Date enddate = null; diff --git a/dmp-backend/pom.xml b/dmp-backend/pom.xml index 2fd661beb..914c54b41 100644 --- a/dmp-backend/pom.xml +++ b/dmp-backend/pom.xml @@ -10,7 +10,7 @@ org.springframework.boot spring-boot-starter-parent - 1.5.9.RELEASE + 2.5.2 @@ -27,25 +27,26 @@ 1.8 0.0.1-SNAPSHOT - 4.3.8.RELEASE + 5.3.8 - 4.2.3.RELEASE - 1.19.1 + 5.3.10.RELEASE + + + - 5.2.11.Final + 5.5.3.Final 1.9 4.11 1.2.17 1.7.12 - 9.0.7.v20131107 - + 1.2.3 - 1 - 3.0.1 - ikalyvas + + @@ -58,7 +59,7 @@ org.springframework spring-context-support - 4.0.0.RELEASE + ${org.springframework.version} org.springframework.boot @@ -139,7 +140,7 @@ com.fasterxml.jackson.dataformat jackson-dataformat-xml - 2.9.3 + 2.12.3 @@ -147,13 +148,13 @@ com.fasterxml.jackson.core jackson-core - 2.9.4 + 2.12.3 com.fasterxml.jackson.core jackson-databind - 2.9.4 + 2.12.3 @@ -263,36 +264,53 @@ 1.3.1 + + + javax.validation + validation-api + 2.0.1.Final + + + io.prometheus simpleclient - 0.10.0 + 0.11.0 io.prometheus simpleclient_hotspot - 0.10.0 + 0.11.0 io.prometheus simpleclient_httpserver - 0.10.0 + 0.11.0 io.prometheus simpleclient_pushgateway - 0.10.0 + 0.11.0 - + - io.prometheus - simpleclient_spring_boot - 0.10.0 + io.micrometer + micrometer-registry-prometheus + 1.7.1 + + + org.springframework.boot + spring-boot-starter-actuator + + + + io.micrometer + micrometer-core diff --git a/dmp-backend/web/pom.xml b/dmp-backend/web/pom.xml index d3cfd31e6..6798b6022 100644 --- a/dmp-backend/web/pom.xml +++ b/dmp-backend/web/pom.xml @@ -50,31 +50,37 @@ com.fasterxml.jackson.core jackson-annotations - 2.9.0 + 2.12.3 - + - + + + com.github.ben-manes.caffeine + caffeine + 2.9.0 + + - junit - junit - 4.12 + org.junit.jupiter + junit-jupiter-api + 5.7.2 test @@ -87,7 +93,7 @@ io.springfox springfox-swagger-ui - 2.7.0 + 3.0.0 @@ -101,7 +107,7 @@ org.springframework.social spring-social-linkedin - 1.0.0.RELEASE + 1.0.2.RELEASE @@ -143,7 +149,6 @@ org.springframework.boot spring-boot-maven-plugin - 1.5.9.RELEASE diff --git a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java index d9c2b65d6..31e87ddf5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java +++ b/dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java @@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; -import org.springframework.boot.web.support.SpringBootServletInitializer; +import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; import org.springframework.scheduling.annotation.EnableAsync; @SpringBootApplication diff --git a/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java b/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java index 0a09a6731..427b61173 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java +++ b/dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java @@ -1,12 +1,12 @@ package eu.eudat.cache; -import com.google.common.cache.CacheBuilder; +import com.github.benmanes.caffeine.cache.Caffeine; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.cache.CacheManager; import org.springframework.cache.annotation.EnableCaching; -import org.springframework.cache.guava.GuavaCache; +import org.springframework.cache.caffeine.CaffeineCache; import org.springframework.cache.support.SimpleCacheManager; import org.springframework.context.annotation.Bean; import org.springframework.stereotype.Component; @@ -29,19 +29,19 @@ public class ResponsesCache { public CacheManager cacheManager() { logger.info("Loading ResponsesCache..."); SimpleCacheManager simpleCacheManager = new SimpleCacheManager(); - List caches = new ArrayList(); - caches.add(new GuavaCache("repositories", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("grants", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("projects", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("funders", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("externalDatasets", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("currencies", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); - caches.add(new GuavaCache("licenses", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + List caches = new ArrayList(); + caches.add(new CaffeineCache("repositories", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("grants", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("projects", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("funders", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("organisations", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("registries", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("services", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("tags", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("researchers", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("externalDatasets", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("currencies", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); + caches.add(new CaffeineCache("licenses", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build())); simpleCacheManager.setCaches(caches); logger.info("OK"); return simpleCacheManager; diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java index 1c5c48bed..a1311f1f1 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java @@ -2,8 +2,8 @@ package eu.eudat.configurations; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder; import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; import org.springframework.context.annotation.*; import org.springframework.core.env.Environment; import org.springframework.dao.annotation.PersistenceExceptionTranslationPostProcessor; diff --git a/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java b/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java index 2c64aa8d1..36d9c502c 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java +++ b/dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java @@ -4,7 +4,6 @@ import eu.eudat.controllers.interceptors.RequestInterceptor; import eu.eudat.logic.handlers.PrincipalArgumentResolver; import eu.eudat.logic.services.ApiContext; import eu.eudat.logic.services.operations.authentication.AuthenticationService; -import io.prometheus.client.spring.boot.EnablePrometheusEndpoint; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableAsync; @@ -18,7 +17,6 @@ import java.util.List; @EnableAsync @Configuration @EnableScheduling -@EnablePrometheusEndpoint public class WebMVCConfiguration extends WebMvcConfigurerAdapter { private ApiContext apiContext; diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java index 26c9cb70a..e27a21053 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java @@ -45,13 +45,15 @@ public class Admin extends BaseController { private DatasetProfileManager datasetProfileManager; private UserManager userManager; private ConfigLoader configLoader; + private final MetricsManager metricsManager; @Autowired - public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader) { + public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader, MetricsManager metricsManager) { super(apiContext); this.datasetProfileManager = datasetProfileManager; this.userManager = userManager; this.configLoader = configLoader; + this.metricsManager = metricsManager; } @Transactional @@ -71,7 +73,7 @@ public class Admin extends BaseController { userDatasetProfile.setRole(0); getApiContext().getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile); datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile); - MetricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL); + metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL); return ResponseEntity.status(HttpStatus.OK).body(modelDefinition.getId()); } @@ -90,7 +92,7 @@ public class Admin extends BaseController { eu.eudat.data.entities.DatasetProfile datasetProfile = this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(datasetprofile); datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile); if (datasetProfile.getStatus() == 1 && oldStatus == 0) { - MetricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.ACTIVE); + metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.ACTIVE); } return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().status(ApiMessageCode.NO_MESSAGE)); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java index f1d773ddf..4821a2b8a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java @@ -65,6 +65,7 @@ public class Login { private ConfigurableProviderTokenValidator configurableProviderTokenValidator; private ZenodoTokenValidator zenodoTokenValidator; private ConfigLoader configLoader; + private final MetricsManager metricsManager; // private Logger logger; @@ -76,7 +77,8 @@ public class Login { B2AccessTokenValidator b2AccessTokenValidator, ORCIDTokenValidator orcidTokenValidator, LinkedInTokenValidator linkedInTokenValidator, OpenAIRETokenValidator openAIRETokenValidator, ConfigurableProviderTokenValidator configurableProviderTokenValidator, ZenodoTokenValidator zenodoTokenValidator, - ConfigLoader configLoader, UserManager userManager) { + ConfigLoader configLoader, UserManager userManager, + MetricsManager metricsManager) { this.customAuthenticationProvider = customAuthenticationProvider; this.nonVerifiedUserAuthenticationService = nonVerifiedUserAuthenticationService; this.twitterTokenValidator = twitterTokenValidator; @@ -88,6 +90,7 @@ public class Login { this.zenodoTokenValidator = zenodoTokenValidator; this.configLoader = configLoader; this.userManager = userManager; + this.metricsManager = metricsManager; } @@ -96,7 +99,7 @@ public class Login { public @ResponseBody ResponseEntity> externallogin(@RequestBody LoginInfo credentials) throws GeneralSecurityException, NullEmailException { logger.info("Trying To Login With " + credentials.getProvider()); - MetricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN); + metricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(customAuthenticationProvider.authenticate(credentials)).status(ApiMessageCode.SUCCESS_MESSAGE)); } @@ -105,7 +108,7 @@ public class Login { public @ResponseBody ResponseEntity> nativelogin(@RequestBody Credentials credentials) throws NullEmailException { logger.info(credentials.getUsername() + " Trying To Login"); - MetricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN); + metricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().payload(userManager.authenticate(this.nonVerifiedUserAuthenticationService, credentials)).status(ApiMessageCode.SUCCESS_MESSAGE)); } @@ -166,7 +169,7 @@ public class Login { ResponseEntity> logout(Principal principal) { this.nonVerifiedUserAuthenticationService.Logout(principal.getToken()); logger.info(principal + " Logged Out"); - MetricsManager.decreaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN); + metricsManager.decreaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN); return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem().status(ApiMessageCode.NO_MESSAGE)); } diff --git a/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java b/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java index f9b519105..2a5dca57a 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java +++ b/dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java @@ -31,16 +31,18 @@ import static eu.eudat.types.Authorities.ADMIN; public class UserGuideController { private Environment environment; + private final MetricsManager metricsManager; @Autowired - public UserGuideController(Environment environment) { + public UserGuideController(Environment environment, MetricsManager metricsManager) { this.environment = environment; + this.metricsManager = metricsManager; } @RequestMapping(path = "{lang}", method = RequestMethod.GET ) public ResponseEntity getUserGuide(@PathVariable(name = "lang") String lang) throws IOException { long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count(); - MetricsManager.calculateValue(MetricNames.LANGUAGES, (int) files, null); + metricsManager.calculateValue(MetricNames.LANGUAGES, (int) files, null); Stream walk = Files.walk(Paths.get(this.environment.getProperty("userguide.path"))); List result = walk.filter(Files::isRegularFile) .map(Path::toString).collect(Collectors.toList()); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java index 10f9cdb4d..2c7eb2fa5 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java @@ -109,9 +109,10 @@ public class DataManagementPlanManager { private Environment environment; private RDAManager rdaManager; private UserManager userManager; + private final MetricsManager metricsManager; @Autowired - public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager) { + public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager, MetricsManager metricsManager) { this.apiContext = apiContext; this.datasetManager = datasetManager; this.utilitiesService = apiContext.getUtilitiesService(); @@ -119,6 +120,7 @@ public class DataManagementPlanManager { this.environment = environment; this.rdaManager = rdaManager; this.userManager = userManager; + this.metricsManager = metricsManager; } /* @@ -150,6 +152,7 @@ public class DataManagementPlanManager { items = null; } } + if (items == null) { items = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dataManagementPlanTableRequest.getCriteria()); } @@ -446,7 +449,7 @@ public class DataManagementPlanManager { setNotification = true; } else { - MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); + metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); } DMP newDmp = dataManagementPlan.toDataModel(); @@ -573,7 +576,7 @@ public class DataManagementPlanManager { if (dataManagementPlan.getStatus() == (int) DMP.DMPStatus.FINALISED.getValue() && dmp1.getStatus().equals(DMP.DMPStatus.FINALISED.getValue())) throw new Exception("DMP is finalized, therefore cannot be edited."); } else { - MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); + metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); } List datasets = new ArrayList<>(); DMP tempDMP = dataManagementPlan.toDataModel(); @@ -633,6 +636,12 @@ public class DataManagementPlanManager { assignGrandUserIfInternal(newDmp, user); assignFunderUserIfInternal(newDmp, user); assignProjectUserIfInternal(newDmp, user); + if (newDmp.getGrant().getStartdate() == null) { + newDmp.getGrant().setStartdate(new Date()); + } + if (newDmp.getGrant().getEnddate() == null) { + newDmp.getGrant().setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS))); + } databaseRepository.getGrantDao().createOrUpdate(newDmp.getGrant()); DMP tempDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp); newDmp.setId(tempDmp.getId()); @@ -701,7 +710,7 @@ public class DataManagementPlanManager { newDmp.setDataset(new HashSet<>(databaseRepository.getDatasetDao().getWithCriteria(criteria1).toList())); this.updateIndex(newDmp); - MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); + metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); return newDmp.getId(); } @@ -715,16 +724,16 @@ public class DataManagementPlanManager { DMP oldDmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(uuid); switch (oldDmp.getStatus()) { case 0: - MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); + metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); break; case 1: if (oldDmp.getDoi() != null) { - MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DOIED); + metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DOIED); } if (oldDmp.isPublic()) { - MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED); + metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED); } - MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED); + metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED); break; } oldDmp.setStatus(DMP.DMPStatus.DELETED.getValue()); @@ -764,7 +773,7 @@ public class DataManagementPlanManager { else { researcher.setCreationUser(user); researcherRepository.createOrUpdate(researcher); - MetricsManager.increaseValue(MetricNames.RESEARCHER, 1, null); + metricsManager.increaseValue(MetricNames.RESEARCHER, 1, null); } } } @@ -805,6 +814,12 @@ public class DataManagementPlanManager { else { grant.setType(Grant.GrantType.EXTERNAL.getValue()); grant.setCreationUser(null); + if (grant.getStartdate() == null) { + grant.setStartdate(new Date()); + } + if (grant.getEnddate() == null) { + grant.setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS))); + } grantDao.createOrUpdate(grant); } } @@ -846,14 +861,14 @@ public class DataManagementPlanManager { project.setType(Project.ProjectType.EXTERNAL.getValue()); if (project.getId() == null) project.setId(UUID.randomUUID()); projectDao.createOrUpdate(project); - MetricsManager.increaseValue(MetricNames.PROJECT, 1, null); + metricsManager.increaseValue(MetricNames.PROJECT, 1, null); } } else { project.setType(Project.ProjectType.EXTERNAL.getValue()); if (project.getId() == null) project.setId(UUID.randomUUID()); projectDao.createOrUpdate(project); - MetricsManager.increaseValue(MetricNames.PROJECT, 1, null); + metricsManager.increaseValue(MetricNames.PROJECT, 1, null); } } } @@ -930,7 +945,7 @@ public class DataManagementPlanManager { return newDataset; }).thenApplyAsync(item -> { futures.add(datasetDao.createOrUpdateAsync(item).whenComplete(((dataset1, throwable) -> { - MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT); + metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT); eu.eudat.elastic.entities.Dataset datasetElastic = new eu.eudat.elastic.entities.Dataset(); datasetElastic.setId(dataset1.getId().toString()); datasetElastic.setLabel(dataset1.getLabel()); @@ -989,7 +1004,7 @@ public class DataManagementPlanManager { dmp.setPublic(true); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp); this.updateIndex(dmp); - MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED); + metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED); DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); criteria.setGroupIds(Collections.singletonList(dmp.getGroupId())); criteria.setAllVersions(true); @@ -1002,7 +1017,7 @@ public class DataManagementPlanManager { tags = elastic.getTags(); } this.datasetManager.updateTags(dataset, tags); - MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED); + metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED); } catch (Exception e) { logger.error(e.getMessage(), e); } @@ -1088,11 +1103,11 @@ public class DataManagementPlanManager { this.updateIndex(dmp); UserInfo user = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal.getId()); sendNotification(dmp, user, NotificationType.DMP_FINALISED); - MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); - MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED); + metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); + metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED); this.updateDatasetsIndex(indexDatasets); - MetricsManager.decreaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.DRAFT); - MetricsManager.increaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.FINALIZED); + metricsManager.decreaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.DRAFT); + metricsManager.increaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.FINALIZED); } public void undoFinalize(UUID id, Principal principal) throws Exception { @@ -1104,8 +1119,8 @@ public class DataManagementPlanManager { dmp.setStatus(DMP.DMPStatus.ACTIVE.getValue()); apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp); this.updateIndex(dmp); - MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED); - MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); + metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED); + metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT); } public void updateUsers(UUID id, List users, Principal principal) throws Exception { @@ -2161,7 +2176,9 @@ public class DataManagementPlanManager { jsonFos.flush(); } fileSystemResource = new FileSystemResource(tempJsonFile); - addFileMapRequest = new HttpEntity<>(fileSystemResource, null); + HttpHeaders jsonHeaders = new HttpHeaders(); + jsonHeaders.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE); + addFileMapRequest = new HttpEntity<>(fileSystemResource, jsonHeaders); String jsonFileName = jsonFile.getHeaders().get("Content-Disposition").get(0).substring(jsonFile.getHeaders().get("Content-Disposition").get(0).lastIndexOf('=') + 1); addFileUrl = links.get("bucket") + "/" + jsonFileName + "?access_token=" + zenodoToken; restTemplate.put(addFileUrl, addFileMapRequest); @@ -2225,51 +2242,6 @@ public class DataManagementPlanManager { } - public long countAllDrafts() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllPublished() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setIsPublic(true); - criteria.setOnlyPublic(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllDoied() { - DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); - criteria.setHasDoi(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); - } - - public long countAllResearchers() { - ResearcherCriteria criteria = new ResearcherCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count(); - } - - public long countAllProjects() { - ProjectCriteria criteria = new ProjectCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count(); - } - - public long countAllFunders() { - FunderCriteria criteria = new FunderCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count(); - } - - public long countAllGrants() { - GrantCriteria criteria = new GrantCriteria(); - return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count(); - } - /*public DataTableData getPublicPaged(DataManagmentPlanPublicTableRequest dataManagementPlanPublicTableRequest, String fieldsGroup, Principal principal) throws Exception { dataManagementPlanPublicTableRequest.setQuery(databaseRepository.getDmpDao().asQueryable().withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class))); QueryableList items = dataManagementPlanPublicTableRequest.applyCriteria(); diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java index 1a9ec21cb..1c8594b45 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java @@ -110,9 +110,10 @@ public class DatasetManager { private UserManager userManager; private ConfigLoader configLoader; private Environment environment; + private final MetricsManager metricsManager; @Autowired - public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment) { + public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment, MetricsManager metricsManager) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository(); @@ -120,6 +121,7 @@ public class DatasetManager { this.userManager = userManager; this.configLoader = configLoader; this.environment = environment; + this.metricsManager = metricsManager; } public DataTableData getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception { @@ -156,17 +158,13 @@ public class DatasetManager { datasetCriteria.setSize(datasetTableRequest.getLength()); List datasets; try { - long start = System.currentTimeMillis(); datasets = datasetRepository.exists() ? datasetRepository.queryIds(datasetCriteria) : null; - long end = System.currentTimeMillis(); - logger.info("Elastic took " + (end - start)/ 1000f + "s"); } catch (Exception ex) { logger.warn(ex.getMessage(), ex); datasets = null; } - long start = System.currentTimeMillis(); UserInfo userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(principal.getId()).build(); // QueryableList items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); QueryableList items; @@ -212,26 +210,10 @@ public class DatasetManager { DataTableData dataTable = new DataTableData<>(); - - /* long countStart = System.currentTimeMillis(); - List pagedIds = pagedItems.toList().stream().map(Dataset::getId).collect(Collectors.toList()); - long countEnd = System.currentTimeMillis(); - - logger.info("Paged Ids took " + (countEnd - countStart)/ 1000f + "s");*/ + dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList())); dataTable.setTotalCount(authItems.count()); - // items = databaseRepository.getDatasetDao().asQueryable().where((builder, root) -> root.get("id").in(pagedIds)).withHint(HintedModelFactory.getHint(DatasetListingModel.class)); - // PaginationManager.applyOrder(items, datasetTableRequest); - long dataStart = System.currentTimeMillis(); - dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList())); - - long dataEnd = System.currentTimeMillis(); - - logger.info("Data retrieval took " + (dataEnd - dataStart)/1000f + "s" ); - - long end = System.currentTimeMillis(); - logger.info("DB took " + (end - start)/ 1000f + "s"); //CompletableFuture.allOf(itemsFuture, countFuture).join(); return dataTable; } @@ -638,7 +620,7 @@ public class DatasetManager { sendNotification = true; } } else { - MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT); + metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT); } if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null) throw new Exception("DMP is finalized, therefore Dataset cannot be edited."); @@ -908,7 +890,7 @@ public class DatasetManager { if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue()) throw new Exception("You cannot make public a Dataset That Has not Been Finalised"); datasetDao.createOrUpdate(dataset); - MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED); + metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED); } public ResponseEntity getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException { @@ -997,7 +979,7 @@ public class DatasetManager { createServicesIfTheyDontExist(entity); createExternalDatasetsIfTheyDontExist(entity); - MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT); + metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT); return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity); } @@ -1112,7 +1094,7 @@ public class DatasetManager { if (!tagNodes.isEmpty()) { tagNodes.forEach(node -> { JsonNode value = node.get("value"); - String stringValue = value.asText().replaceAll("=", ":"); + String stringValue = value.toString().replaceAll("=", ":"); JSONArray values = new JSONArray(stringValue); if (values != null) { values.iterator().forEachRemaining(element -> { @@ -1155,28 +1137,4 @@ public class DatasetManager { listingModel.setProfileLatestVersion(islast);*/ return listingModel; } - - public long countAllDraft() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllPublic() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setIsPublic(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } - - public long countAllWithDoi() { - eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); - criteria.setHasDoi(true); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); - } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java index 049aac08d..0cd4b53a3 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java @@ -60,12 +60,14 @@ public class DatasetProfileManager { private DatabaseRepository databaseRepository; private Environment environment; private List cache; + private final MetricsManager metricsManager; @Autowired - public DatasetProfileManager(ApiContext apiContext, Environment environment) { + public DatasetProfileManager(ApiContext apiContext, Environment environment, MetricsManager metricsManager) { this.apiContext = apiContext; this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository(); this.environment = environment; + this.metricsManager = metricsManager; this.cache = new ArrayList<>(); } @@ -93,7 +95,7 @@ public class DatasetProfileManager { DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(UUID.fromString(id)); apiContext.getOperationsContext().getDatabaseRepository().detachEntity(profile); profile.setId(null); - MetricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL); + metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL); return profile; } @@ -234,7 +236,7 @@ public class DatasetProfileManager { File localFile = convert(multiPartFile); eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileModel.DatasetProfile profile = xmlBuilder.build(localFile); Files.deleteIfExists(localFile.toPath()); - MetricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL); + metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL); return profile; } catch (IOException e) { logger.error(e.getMessage(), e); @@ -317,25 +319,4 @@ public class DatasetProfileManager { }).collect(Collectors.toList())); } } - - public long countAlldraft() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(0); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); - } - - public long countAllFinalized() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(1); - return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); - } - - @Transactional - public long countAllUsed() { - DatasetProfileCriteria criteria = new DatasetProfileCriteria(); - criteria.setStatus(1); - criteria.setAllVersions(false); - List datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList(); - return datasetProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).filter(datasetProfile -> !datasetProfile.getDataset().isEmpty()).count(); - } } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java index 7e39b9dec..de5b54006 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java @@ -1,6 +1,12 @@ package eu.eudat.logic.managers; import eu.eudat.types.MetricNames; +import eu.eudat.data.dao.criteria.*; +import eu.eudat.data.entities.Dataset; +import eu.eudat.data.entities.DatasetProfile; +import eu.eudat.logic.services.ApiContext; +import eu.eudat.logic.utilities.helpers.StreamDistinctBy; +import io.micrometer.prometheus.PrometheusMeterRegistry; import io.prometheus.client.Gauge; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -14,32 +20,16 @@ import javax.transaction.Transactional; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import java.util.stream.Stream; @Component public class MetricsManager { private final static Logger logger = LoggerFactory.getLogger(MetricsManager.class); - private static Map gauges = Stream.of( new Object[][]{ - {MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register()}, + private final Map gauges; - {MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register()}, - {MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register()}, - {MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register()}, - {MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register()}, - - {MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register()}, - - {MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register()}, - - {MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register()}, - - {MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register()}, - - }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1])); - - public static void increaseValue(String name, int amount, String label) { + public void increaseValue(String name, int amount, String label) { if(label != null) { gauges.get(name).labels(label).inc(amount); @@ -48,7 +38,7 @@ public class MetricsManager { } } - public static void decreaseValue(String name, int amount, String label) { + public void decreaseValue(String name, int amount, String label) { if(label != null) { gauges.get(name).labels(label).dec(amount); } else { @@ -56,7 +46,7 @@ public class MetricsManager { } } - public static Integer getValue(String name, String label) { + public Integer getValue(String name, String label) { if(label != null) { return Double.valueOf(gauges.get(name).labels(label).get()).intValue(); } else { @@ -64,7 +54,7 @@ public class MetricsManager { } } - public static void calculateValue(String name, int amount, String label) { + public void calculateValue(String name, int amount, String label) { Integer orig = getValue(name, label); int diff = orig - amount; if (diff != 0) { @@ -76,19 +66,34 @@ public class MetricsManager { } } - private final DatasetManager datasetManager; - private final DataManagementPlanManager dataManagementPlanManager; - private final DatasetProfileManager datasetProfileManager; + private final ApiContext apiContext; private final UserManager userManager; private final Environment environment; @Autowired - public MetricsManager(DatasetManager datasetManager, DataManagementPlanManager dataManagementPlanManager, DatasetProfileManager datasetProfileManager, UserManager userManager, Environment environment) { - this.datasetManager = datasetManager; - this.dataManagementPlanManager = dataManagementPlanManager; - this.datasetProfileManager = datasetProfileManager; + public MetricsManager(ApiContext apiContext, UserManager userManager, Environment environment, PrometheusMeterRegistry registry) { + this.apiContext = apiContext; this.userManager = userManager; this.environment = environment; + registry.clear(); + this.gauges = Stream.of( new Object[][]{ + {MetricNames.DMP, Gauge.build().name("argos_managed_dmps").help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())}, + + {MetricNames.FUNDERS, Gauge.build().name("argos_funders").help("Number of registered Funders").register(registry.getPrometheusRegistry())}, + {MetricNames.GRANTS, Gauge.build().name("argos_grants").help("Number of registered Grants").register(registry.getPrometheusRegistry())}, + {MetricNames.PROJECT, Gauge.build().name("argos_projects").help("Number of registered Projects").register(registry.getPrometheusRegistry())}, + {MetricNames.RESEARCHER, Gauge.build().name("argos_researchers").help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())}, + + {MetricNames.DATASET, Gauge.build().name("argos_managed_dataset_descriptions").help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())}, + + {MetricNames.DATASET_TEMPLATE, Gauge.build().name("argos_dataset_templates").help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())}, + + {MetricNames.USERS, Gauge.build().name("argos_users").help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())}, + + {MetricNames.LANGUAGES, Gauge.build().name("argos_languages").help("Number of Languages").register(registry.getPrometheusRegistry())}, + + }).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1])); + } @PostConstruct @@ -96,32 +101,131 @@ public class MetricsManager { @Scheduled(initialDelay = 1000 * 60 * 60, fixedDelay = 1000 * 60 * 60) public void init() throws IOException { logger.info("Start calculating Metrics"); - calculateValue(MetricNames.DMP, (int) dataManagementPlanManager.countAllDrafts(), MetricNames.DRAFT); - calculateValue(MetricNames.DMP, (int) dataManagementPlanManager.countAllFinalized(), MetricNames.FINALIZED); - calculateValue(MetricNames.DMP, (int) dataManagementPlanManager.countAllPublished(), MetricNames.PUBLISHED); - calculateValue(MetricNames.DMP, (int) dataManagementPlanManager.countAllDoied(), MetricNames.DOIED); + calculateValue("argos_managed_dmps", (int) countAllDraftDMPs(), "draft"); + calculateValue("argos_managed_dmps", (int) countAllFinalizedDMPs(), "finalized"); + calculateValue("argos_managed_dmps", (int) countAllPublishedDMPs(), "published"); + calculateValue("argos_managed_dmps", (int) countAllDoiedDMPs(), "doied"); - calculateValue(MetricNames.FUNDERS, (int) dataManagementPlanManager.countAllFunders(), null); - calculateValue(MetricNames.GRANTS, (int) dataManagementPlanManager.countAllGrants(), null); - calculateValue(MetricNames.PROJECT, (int) dataManagementPlanManager.countAllProjects(), null); - calculateValue(MetricNames.RESEARCHER, (int) dataManagementPlanManager.countAllResearchers(), null); + calculateValue("argos_funders", (int) countAllFunders(), null); + calculateValue("argos_grants", (int) countAllGrants(), null); + calculateValue("argos_projects", (int) countAllProjects(), null); + calculateValue("argos_researchers", (int) countAllResearchers(), null); - calculateValue(MetricNames.DATASET, (int) datasetManager.countAllDraft(), MetricNames.DRAFT); - calculateValue(MetricNames.DATASET, (int) datasetManager.countAllFinalized(), MetricNames.FINALIZED); - calculateValue(MetricNames.DATASET, (int) datasetManager.countAllPublic(), MetricNames.PUBLISHED); - calculateValue(MetricNames.DATASET, (int) datasetManager.countAllWithDoi(), MetricNames.DOIED); + calculateValue("argos_managed_dataset_descriptions", (int) countAllDraftDatasets(), "draft"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllFinalizedDatasets(), "finalized"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllPublicDatasets(), "published"); + calculateValue("argos_managed_dataset_descriptions", (int) countAllDatasetsWithDoi(), "doied"); - calculateValue(MetricNames.DATASET_TEMPLATE, (int) datasetProfileManager.countAlldraft(), MetricNames.DRAFT); - calculateValue(MetricNames.DATASET_TEMPLATE, (int) datasetProfileManager.countAllFinalized(), MetricNames.ACTIVE); - calculateValue(MetricNames.DATASET_TEMPLATE, (int) datasetProfileManager.countAllUsed(), MetricNames.USED); + calculateValue("argos_dataset_templates", (int) countAllDraftTemplates(), "draft"); + calculateValue("argos_dataset_templates", (int) countAllFinalizedTemplates(), "active"); + calculateValue("argos_dataset_templates", (int) countAllUsedTemplates(), "used"); - calculateValue(MetricNames.USERS, (int) userManager.countActiveUsers().intValue(), MetricNames.LOGGEDIN); - calculateValue(MetricNames.USERS, (int) userManager.countAllUsers().intValue(), MetricNames.TOTAL); + calculateValue("argos_users", (int) userManager.countActiveUsers().intValue(), "loggedin"); + calculateValue("argos_users", (int) userManager.countAllUsers().intValue(), "total"); long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count(); - calculateValue(MetricNames.LANGUAGES, (int) files, null); + calculateValue("argos_languages", (int) files, null); logger.info("Metrics calculation Completed"); } + private long countAllDraftDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllFinalizedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllPublishedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setIsPublic(true); + criteria.setOnlyPublic(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllDoiedDMPs() { + DataManagementPlanCriteria criteria = new DataManagementPlanCriteria(); + criteria.setHasDoi(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count(); + } + + private long countAllResearchers() { + ResearcherCriteria criteria = new ResearcherCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count(); + } + + private long countAllProjects() { + ProjectCriteria criteria = new ProjectCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count(); + } + + private long countAllFunders() { + FunderCriteria criteria = new FunderCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count(); + } + + private long countAllGrants() { + GrantCriteria criteria = new GrantCriteria(); + return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count(); + } + + public long countAllDraftDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllFinalizedDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllPublicDatasets() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setIsPublic(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllDatasetsWithDoi() { + eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria(); + criteria.setHasDoi(true); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count(); + } + + public long countAllDraftTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(0); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); + } + + public long countAllFinalizedTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(1); + return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count(); + } + + @Transactional + public long countAllUsedTemplates() { + DatasetProfileCriteria criteria = new DatasetProfileCriteria(); + criteria.setStatus(1); + criteria.setAllVersions(false); + List datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList(); + List filteredProfiles = new ArrayList<>(); + for (DatasetProfile datasetProfile : datasetProfiles) { + DatasetCriteria datasetCriteria = new DatasetCriteria(); + datasetCriteria.setDatasetTemplates(Collections.singletonList(datasetProfile.getId())); + Long datasetCount = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria).count(); + if (datasetCount > 0) { + filteredProfiles.add(datasetProfile); + } + } + return filteredProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).count(); + } + } diff --git a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java index a5a4c293d..7ed37c941 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java +++ b/dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java @@ -459,7 +459,7 @@ public class RemoteFetcher { try { String filePath = Paths.get(path).toUri().toURL().toString(); ObjectMapper mapper = new ObjectMapper(); - internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); + internalResults = mapper.readValue(new File(filePath), new TypeReference>>(){}); return searchListMap(internalResults, query); } catch (Exception e) { logger.error(e.getMessage(), e); diff --git a/dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java b/dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java index 46c193154..eae784aeb 100644 --- a/dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java +++ b/dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java @@ -7,7 +7,10 @@ import eu.eudat.models.data.funder.FunderDMPEditorModel; import eu.eudat.models.data.grant.GrantDMPEditorModel; import eu.eudat.models.data.project.ProjectDMPEditorModel; import eu.eudat.models.data.userinfo.UserInfo; +import eu.eudat.models.data.userinfo.UserListingModel; import net.minidev.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.*; import java.util.stream.Collectors; @@ -16,6 +19,8 @@ import java.util.stream.Collectors; * Created by ikalyvas on 2/5/2018. */ public class DataManagementPlanNewVersionModel implements DataModel { + private static final Logger logger = LoggerFactory.getLogger(DataManagementPlanNewVersionModel.class); + private UUID id; private String label; private UUID groupId; @@ -26,7 +31,7 @@ public class DataManagementPlanNewVersionModel implements DataModel organisations; private List researchers; - private List associatedUsers; + private List associatedUsers; private eu.eudat.models.data.userinfo.UserInfo creator; private Date created; private List datasets; @@ -104,10 +109,10 @@ public class DataManagementPlanNewVersionModel implements DataModel getAssociatedUsers() { + public List getAssociatedUsers() { return associatedUsers; } - public void setAssociatedUsers(List associatedUsers) { + public void setAssociatedUsers(List associatedUsers) { this.associatedUsers = associatedUsers; } @@ -163,7 +168,7 @@ public class DataManagementPlanNewVersionModel implements DataModel(new ArrayList<>(this.associatedUsers))); + // entity.setUsers(new HashSet<>(new ArrayList<>(this.associatedUsers))); entity.setDescription(this.description); entity.setStatus((short) this.status); entity.setGroupId(this.groupId == null ? UUID.randomUUID() : this.groupId); @@ -227,7 +232,7 @@ public class DataManagementPlanNewVersionModel implements DataModel