MaDgiK-CITE
/
argos
13
0
Fork 0
Code Issues Pull Requests Projects Releases 3 Wiki Activity

Merge branch 'spring-update' into Development 

# Conflicts:
#	dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java
#	dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java
#	dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java
#	dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java
This commit is contained in:
George Kalampokis 2021-10-08 16:04:52 +03:00
parent 0e97e6c0a8 1ae2788453
commit a3af8ce3a5
17 changed files with 321 additions and 259 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dmp-backend/data/src/main/java/eu/eudat/data/entities/Grant.java
View File

@ -106,11 +106,11 @@ public class Grant implements DataEntity<Grant, UUID> {
@Column(name = "\"Definition\"", columnDefinition = "xml", nullable = true)
private String definition;
@Column(name = "\"StartDate\"", nullable = false)
@Column(name = "\"StartDate\"", nullable = true)
@Convert(converter = DateToUTCConverter.class)
private Date startdate = null;
@Column(name = "\"EndDate\"", nullable = false)
@Column(name = "\"EndDate\"", nullable = true)
@Convert(converter = DateToUTCConverter.class)
private Date enddate = null;

64
dmp-backend/pom.xml
View File

@ -10,7 +10,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.9.RELEASE</version>
<version>2.5.2</version>
</parent>
<modules>
@ -27,25 +27,26 @@
<java.version>1.8</java.version>
<dmp-backend-commons.version>0.0.1-SNAPSHOT</dmp-backend-commons.version>
<org.springframework.version>4.3.8.RELEASE</org.springframework.version>
<org.springframework.version>5.3.8</org.springframework.version>
<!-- <org.springframeweu.eudat.logic.securityrity.version>3.2.10.RELEASE</org.springframeweu.eudat.logic.securityrity.version> -->
<org.springframework.security.version>4.2.3.RELEASE</org.springframework.security.version>
<com.sun.jersey.version>1.19.1</com.sun.jersey.version>
<org.springframework.security.version>5.3.10.RELEASE</org.springframework.security.version>
<!--<com.sun.jersey.version>1.19.1</com.sun.jersey.version>-->
<!--
<org.apache.tomcat.tomcat-jdbc.version>7.0.35</org.apache.tomcat.tomcat-jdbc.version>
<com.fasterxml.jackson>2.8.4</com.fasterxml.jackson>
-->
<!--<com.fasterxml.jackson>2.9.0</com.fasterxml.jackson>-->
<hibernate.version>5.2.11.Final</hibernate.version>
<hibernate.version>5.5.3.Final</hibernate.version>
<commons-codec.version>1.9</commons-codec.version>
<org.junit.version>4.11</org.junit.version>
<log4j.version>1.2.17</log4j.version>
<slf4j.version>1.7.12</slf4j.version>
<jetty.version>9.0.7.v20131107
</jetty.version> <!-- Adapt this to a version found on http://repo.maven.apache.org/maven2/org/eclipse/jetty/jetty-maven-plugin/ -->
<!--<jetty.version>11.0.5
</jetty.version>--> <!-- Adapt this to a version found on http://repo.maven.apache.org/maven2/org/eclipse/jetty/jetty-maven-plugin/ -->
<logback.version>1.2.3</logback.version>
<javax.inject.version>1</javax.inject.version>
<javax.servlet.servlet-api.version>3.0.1</javax.servlet.servlet-api.version>
<docker.image.prefix>ikalyvas</docker.image.prefix>
<!-- <javax.inject.version>1</javax.inject.version>-->
<!--<javax.servlet.servlet-api.version>3.0.1</javax.servlet.servlet-api.version>-->
</properties>
<dependencies>
@ -58,7 +59,7 @@
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context-support</artifactId>
<version>4.0.0.RELEASE</version>
<version>${org.springframework.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
@ -139,7 +140,7 @@
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-xml</artifactId>
<version>2.9.3</version>
<version>2.12.3</version>
</dependency>
@ -147,13 +148,13 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>2.9.4</version>
<version>2.12.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.fasterxml.jackeu.eudat.corecore/jackson-databind -->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.4</version>
<version>2.12.3</version>
</dependency>
<!-- g/a spring -->
@ -263,36 +264,53 @@
<version>1.3.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/javax.validation/validation-api -->
<dependency>
<groupId>javax.validation</groupId>
<artifactId>validation-api</artifactId>
<version>2.0.1.Final</version>
</dependency>
<!-- The client -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient</artifactId>
<version>0.10.0</version>
<version>0.11.0</version>
</dependency>
<!-- Hotspot JVM metrics-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_hotspot</artifactId>
<version>0.10.0</version>
<version>0.11.0</version>
</dependency>
<!-- Exposition HTTPServer-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_httpserver</artifactId>
<version>0.10.0</version>
<version>0.11.0</version>
</dependency>
<!-- Pushgateway exposition-->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_pushgateway</artifactId>
<version>0.10.0</version>
<version>0.11.0</version>
</dependency>
<!-- Pushgateway exposition-->
<!-- https://mvnrepository.com/artifact/io.micrometer/micrometer-registry-prometheus -->
<dependency>
<groupId>io.prometheus</groupId>
<artifactId>simpleclient_spring_boot</artifactId>
<version>0.10.0</version>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-registry-prometheus</artifactId>
<version>1.7.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>
<!-- Micormeter core dependecy -->
<dependency>
<groupId>io.micrometer</groupId>
<artifactId>micrometer-core</artifactId>
</dependency>
</dependencies>

31
dmp-backend/web/pom.xml
View File

@ -50,31 +50,37 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>2.9.0</version>
<version>2.12.3</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework/spring-context -->
<dependency>
<!--<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
<version>4.3.10.RELEASE</version>
</dependency>
<version>5.3.8</version>
</dependency>-->
<!-- guava eu.eudat.cache -->
<dependency>
<!-- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>23.0</version>
</dependency>
<version>30.1.1-jre</version>
</dependency>-->
<!-- https://mvnrepository.com/artifact/org.springframework/spring-context-support -->
<dependency>
<groupId>com.github.ben-manes.caffeine</groupId>
<artifactId>caffeine</artifactId>
<version>2.9.0</version>
</dependency>
<!-- https://mvnrepository.com/artifact/junit/junit -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.12</version>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.7.2</version>
<scope>test</scope>
</dependency>
@ -87,7 +93,7 @@
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>2.7.0</version>
<version>3.0.0</version>
</dependency>
<!-- facebook Login -->
@ -101,7 +107,7 @@
<dependency>
<groupId>org.springframework.social</groupId>
<artifactId>spring-social-linkedin</artifactId>
<version>1.0.0.RELEASE</version>
<version>1.0.2.RELEASE</version>
</dependency>
<!-- tweeter login-->
@ -143,7 +149,6 @@
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>1.5.9.RELEASE</version>
</plugin>
</plugins>
</build>

2
dmp-backend/web/src/main/java/eu/eudat/EuDatApplication.java
View File

@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.web.support.SpringBootServletInitializer;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.scheduling.annotation.EnableAsync;
@SpringBootApplication

30
dmp-backend/web/src/main/java/eu/eudat/cache/ResponsesCache.java vendored
View File

@ -1,12 +1,12 @@
package eu.eudat.cache;
import com.google.common.cache.CacheBuilder;
import com.github.benmanes.caffeine.cache.Caffeine;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.cache.CacheManager;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.cache.guava.GuavaCache;
import org.springframework.cache.caffeine.CaffeineCache;
import org.springframework.cache.support.SimpleCacheManager;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component;
@ -29,19 +29,19 @@ public class ResponsesCache {
public CacheManager cacheManager() {
logger.info("Loading ResponsesCache...");
SimpleCacheManager simpleCacheManager = new SimpleCacheManager();
List<GuavaCache> caches = new ArrayList<GuavaCache>();
caches.add(new GuavaCache("repositories", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("grants", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("projects", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("funders", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("organisations", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("registries", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("services", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("tags", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("researchers", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("externalDatasets", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("currencies", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new GuavaCache("licenses", CacheBuilder.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
List<CaffeineCache> caches = new ArrayList<CaffeineCache>();
caches.add(new CaffeineCache("repositories", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("grants", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("projects", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("funders", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("organisations", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("registries", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("services", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("tags", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("researchers", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("externalDatasets", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("currencies", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
caches.add(new CaffeineCache("licenses", Caffeine.newBuilder().expireAfterAccess(HOW_MANY, TIME_UNIT).build()));
simpleCacheManager.setCaches(caches);
logger.info("OK");
return simpleCacheManager;

2
dmp-backend/web/src/main/java/eu/eudat/configurations/DevelDatabaseConfiguration.java
View File

@ -2,8 +2,8 @@ package eu.eudat.configurations;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.jdbc.DataSourceBuilder;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.jdbc.DataSourceBuilder;
import org.springframework.context.annotation.*;
import org.springframework.core.env.Environment;
import org.springframework.dao.annotation.PersistenceExceptionTranslationPostProcessor;

2
dmp-backend/web/src/main/java/eu/eudat/configurations/WebMVCConfiguration.java
View File

@ -4,7 +4,6 @@ import eu.eudat.controllers.interceptors.RequestInterceptor;
import eu.eudat.logic.handlers.PrincipalArgumentResolver;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.services.operations.authentication.AuthenticationService;
import io.prometheus.client.spring.boot.EnablePrometheusEndpoint;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.scheduling.annotation.EnableAsync;
@ -18,7 +17,6 @@ import java.util.List;
@EnableAsync
@Configuration
@EnableScheduling
@EnablePrometheusEndpoint
public class WebMVCConfiguration extends WebMvcConfigurerAdapter {
private ApiContext apiContext;

8
dmp-backend/web/src/main/java/eu/eudat/controllers/Admin.java
View File

@ -45,13 +45,15 @@ public class Admin extends BaseController {
private DatasetProfileManager datasetProfileManager;
private UserManager userManager;
private ConfigLoader configLoader;
private final MetricsManager metricsManager;
@Autowired
public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader) {
public Admin(ApiContext apiContext, DatasetProfileManager datasetProfileManager, UserManager userManager/*, Logger logger*/, ConfigLoader configLoader, MetricsManager metricsManager) {
super(apiContext);
this.datasetProfileManager = datasetProfileManager;
this.userManager = userManager;
this.configLoader = configLoader;
this.metricsManager = metricsManager;
}
@Transactional
@ -71,7 +73,7 @@ public class Admin extends BaseController {
userDatasetProfile.setRole(0);
getApiContext().getOperationsContext().getDatabaseRepository().getUserDatasetProfileDao().createOrUpdate(userDatasetProfile);
datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile);
MetricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
return ResponseEntity.status(HttpStatus.OK).body(modelDefinition.getId());
}
@ -90,7 +92,7 @@ public class Admin extends BaseController {
eu.eudat.data.entities.DatasetProfile datasetProfile = this.getApiContext().getOperationsContext().getDatabaseRepository().getDatasetProfileDao().createOrUpdate(datasetprofile);
datasetProfileManager.storeDatasetProfileUsers(datasetProfile, profile);
if (datasetProfile.getStatus() == 1 && oldStatus == 0) {
MetricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.ACTIVE);
metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.ACTIVE);
}
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<UUID>().status(ApiMessageCode.NO_MESSAGE));
}

11
dmp-backend/web/src/main/java/eu/eudat/controllers/Login.java
View File

@ -65,6 +65,7 @@ public class Login {
private ConfigurableProviderTokenValidator configurableProviderTokenValidator;
private ZenodoTokenValidator zenodoTokenValidator;
private ConfigLoader configLoader;
private final MetricsManager metricsManager;
// private Logger logger;
@ -76,7 +77,8 @@ public class Login {
B2AccessTokenValidator b2AccessTokenValidator, ORCIDTokenValidator orcidTokenValidator,
LinkedInTokenValidator linkedInTokenValidator, OpenAIRETokenValidator openAIRETokenValidator,
ConfigurableProviderTokenValidator configurableProviderTokenValidator, ZenodoTokenValidator zenodoTokenValidator,
ConfigLoader configLoader, UserManager userManager) {
ConfigLoader configLoader, UserManager userManager,
MetricsManager metricsManager) {
this.customAuthenticationProvider = customAuthenticationProvider;
this.nonVerifiedUserAuthenticationService = nonVerifiedUserAuthenticationService;
this.twitterTokenValidator = twitterTokenValidator;
@ -88,6 +90,7 @@ public class Login {
this.zenodoTokenValidator = zenodoTokenValidator;
this.configLoader = configLoader;
this.userManager = userManager;
this.metricsManager = metricsManager;
}
@ -96,7 +99,7 @@ public class Login {
public @ResponseBody
ResponseEntity<ResponseItem<PrincipalModel>> externallogin(@RequestBody LoginInfo credentials) throws GeneralSecurityException, NullEmailException {
logger.info("Trying To Login With " + credentials.getProvider());
MetricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
metricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<PrincipalModel>().payload(customAuthenticationProvider.authenticate(credentials)).status(ApiMessageCode.SUCCESS_MESSAGE));
}
@ -105,7 +108,7 @@ public class Login {
public @ResponseBody
ResponseEntity<ResponseItem<PrincipalModel>> nativelogin(@RequestBody Credentials credentials) throws NullEmailException {
logger.info(credentials.getUsername() + " Trying To Login");
MetricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
metricsManager.increaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<PrincipalModel>().payload(userManager.authenticate(this.nonVerifiedUserAuthenticationService, credentials)).status(ApiMessageCode.SUCCESS_MESSAGE));
}
@ -166,7 +169,7 @@ public class Login {
ResponseEntity<ResponseItem<Principal>> logout(Principal principal) {
this.nonVerifiedUserAuthenticationService.Logout(principal.getToken());
logger.info(principal + " Logged Out");
MetricsManager.decreaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
metricsManager.decreaseValue(MetricNames.USERS, 1, MetricNames.LOGGEDIN);
return ResponseEntity.status(HttpStatus.OK).body(new ResponseItem<Principal>().status(ApiMessageCode.NO_MESSAGE));
}

6
dmp-backend/web/src/main/java/eu/eudat/controllers/UserGuideController.java
View File

@ -31,16 +31,18 @@ import static eu.eudat.types.Authorities.ADMIN;
public class UserGuideController {
private Environment environment;
private final MetricsManager metricsManager;
@Autowired
public UserGuideController(Environment environment) {
public UserGuideController(Environment environment, MetricsManager metricsManager) {
this.environment = environment;
this.metricsManager = metricsManager;
}
@RequestMapping(path = "{lang}", method = RequestMethod.GET )
public ResponseEntity getUserGuide(@PathVariable(name = "lang") String lang) throws IOException {
long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count();
MetricsManager.calculateValue(MetricNames.LANGUAGES, (int) files, null);
metricsManager.calculateValue(MetricNames.LANGUAGES, (int) files, null);
Stream<Path> walk = Files.walk(Paths.get(this.environment.getProperty("userguide.path")));
List<String> result = walk.filter(Files::isRegularFile)
.map(Path::toString).collect(Collectors.toList());

104
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DataManagementPlanManager.java
View File

@ -109,9 +109,10 @@ public class DataManagementPlanManager {
private Environment environment;
private RDAManager rdaManager;
private UserManager userManager;
private final MetricsManager metricsManager;
@Autowired
public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager) {
public DataManagementPlanManager(ApiContext apiContext, DatasetManager datasetManager, Environment environment, RDAManager rdaManager, UserManager userManager, MetricsManager metricsManager) {
this.apiContext = apiContext;
this.datasetManager = datasetManager;
this.utilitiesService = apiContext.getUtilitiesService();
@ -119,6 +120,7 @@ public class DataManagementPlanManager {
this.environment = environment;
this.rdaManager = rdaManager;
this.userManager = userManager;
this.metricsManager = metricsManager;
}
/*
@ -150,6 +152,7 @@ public class DataManagementPlanManager {
items = null;
}
}
if (items == null) {
items = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(dataManagementPlanTableRequest.getCriteria());
}
@ -446,7 +449,7 @@ public class DataManagementPlanManager {
setNotification = true;
} else {
MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
}
DMP newDmp = dataManagementPlan.toDataModel();
@ -573,7 +576,7 @@ public class DataManagementPlanManager {
if (dataManagementPlan.getStatus() == (int) DMP.DMPStatus.FINALISED.getValue() && dmp1.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()))
throw new Exception("DMP is finalized, therefore cannot be edited.");
} else {
MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
}
List<Dataset> datasets = new ArrayList<>();
DMP tempDMP = dataManagementPlan.toDataModel();
@ -633,6 +636,12 @@ public class DataManagementPlanManager {
assignGrandUserIfInternal(newDmp, user);
assignFunderUserIfInternal(newDmp, user);
assignProjectUserIfInternal(newDmp, user);
if (newDmp.getGrant().getStartdate() == null) {
newDmp.getGrant().setStartdate(new Date());
}
if (newDmp.getGrant().getEnddate() == null) {
newDmp.getGrant().setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS)));
}
databaseRepository.getGrantDao().createOrUpdate(newDmp.getGrant());
DMP tempDmp = databaseRepository.getDmpDao().createOrUpdate(newDmp);
newDmp.setId(tempDmp.getId());
@ -701,7 +710,7 @@ public class DataManagementPlanManager {
newDmp.setDataset(new HashSet<>(databaseRepository.getDatasetDao().getWithCriteria(criteria1).toList()));
this.updateIndex(newDmp);
MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
return newDmp.getId();
}
@ -715,16 +724,16 @@ public class DataManagementPlanManager {
DMP oldDmp = apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().find(uuid);
switch (oldDmp.getStatus()) {
case 0:
MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
break;
case 1:
if (oldDmp.getDoi() != null) {
MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DOIED);
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DOIED);
}
if (oldDmp.isPublic()) {
MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED);
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED);
}
MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
break;
}
oldDmp.setStatus(DMP.DMPStatus.DELETED.getValue());
@ -764,7 +773,7 @@ public class DataManagementPlanManager {
else {
researcher.setCreationUser(user);
researcherRepository.createOrUpdate(researcher);
MetricsManager.increaseValue(MetricNames.RESEARCHER, 1, null);
metricsManager.increaseValue(MetricNames.RESEARCHER, 1, null);
}
}
}
@ -805,6 +814,12 @@ public class DataManagementPlanManager {
else {
grant.setType(Grant.GrantType.EXTERNAL.getValue());
grant.setCreationUser(null);
if (grant.getStartdate() == null) {
grant.setStartdate(new Date());
}
if (grant.getEnddate() == null) {
grant.setEnddate(Date.from(Instant.now().plus(365, ChronoUnit.DAYS)));
}
grantDao.createOrUpdate(grant);
}
}
@ -846,14 +861,14 @@ public class DataManagementPlanManager {
project.setType(Project.ProjectType.EXTERNAL.getValue());
if (project.getId() == null) project.setId(UUID.randomUUID());
projectDao.createOrUpdate(project);
MetricsManager.increaseValue(MetricNames.PROJECT, 1, null);
metricsManager.increaseValue(MetricNames.PROJECT, 1, null);
}
}
else {
project.setType(Project.ProjectType.EXTERNAL.getValue());
if (project.getId() == null) project.setId(UUID.randomUUID());
projectDao.createOrUpdate(project);
MetricsManager.increaseValue(MetricNames.PROJECT, 1, null);
metricsManager.increaseValue(MetricNames.PROJECT, 1, null);
}
}
}
@ -930,7 +945,7 @@ public class DataManagementPlanManager {
return newDataset;
}).thenApplyAsync(item -> {
futures.add(datasetDao.createOrUpdateAsync(item).whenComplete(((dataset1, throwable) -> {
MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
eu.eudat.elastic.entities.Dataset datasetElastic = new eu.eudat.elastic.entities.Dataset();
datasetElastic.setId(dataset1.getId().toString());
datasetElastic.setLabel(dataset1.getLabel());
@ -989,7 +1004,7 @@ public class DataManagementPlanManager {
dmp.setPublic(true);
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp);
this.updateIndex(dmp);
MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED);
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.PUBLISHED);
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setGroupIds(Collections.singletonList(dmp.getGroupId()));
criteria.setAllVersions(true);
@ -1002,7 +1017,7 @@ public class DataManagementPlanManager {
tags = elastic.getTags();
}
this.datasetManager.updateTags(dataset, tags);
MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
@ -1088,11 +1103,11 @@ public class DataManagementPlanManager {
this.updateIndex(dmp);
UserInfo user = apiContext.getOperationsContext().getDatabaseRepository().getUserInfoDao().find(principal.getId());
sendNotification(dmp, user, NotificationType.DMP_FINALISED);
MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
this.updateDatasetsIndex(indexDatasets);
MetricsManager.decreaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.DRAFT);
MetricsManager.increaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.FINALIZED);
metricsManager.decreaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DATASET, indexDatasets.size(), MetricNames.FINALIZED);
}
public void undoFinalize(UUID id, Principal principal) throws Exception {
@ -1104,8 +1119,8 @@ public class DataManagementPlanManager {
dmp.setStatus(DMP.DMPStatus.ACTIVE.getValue());
apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().createOrUpdate(dmp);
this.updateIndex(dmp);
MetricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
MetricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
metricsManager.decreaseValue(MetricNames.DMP, 1, MetricNames.FINALIZED);
metricsManager.increaseValue(MetricNames.DMP, 1, MetricNames.DRAFT);
}
public void updateUsers(UUID id, List<UserInfoListingModel> users, Principal principal) throws Exception {
@ -2161,7 +2176,9 @@ public class DataManagementPlanManager {
jsonFos.flush();
}
fileSystemResource = new FileSystemResource(tempJsonFile);
addFileMapRequest = new HttpEntity<>(fileSystemResource, null);
HttpHeaders jsonHeaders = new HttpHeaders();
jsonHeaders.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_OCTET_STREAM_VALUE);
addFileMapRequest = new HttpEntity<>(fileSystemResource, jsonHeaders);
String jsonFileName = jsonFile.getHeaders().get("Content-Disposition").get(0).substring(jsonFile.getHeaders().get("Content-Disposition").get(0).lastIndexOf('=') + 1);
addFileUrl = links.get("bucket") + "/" + jsonFileName + "?access_token=" + zenodoToken;
restTemplate.put(addFileUrl, addFileMapRequest);
@ -2225,51 +2242,6 @@ public class DataManagementPlanManager {
}
public long countAllDrafts() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
public long countAllFinalized() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
public long countAllPublished() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
public long countAllDoied() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
public long countAllResearchers() {
ResearcherCriteria criteria = new ResearcherCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count();
}
public long countAllProjects() {
ProjectCriteria criteria = new ProjectCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count();
}
public long countAllFunders() {
FunderCriteria criteria = new FunderCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count();
}
public long countAllGrants() {
GrantCriteria criteria = new GrantCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count();
}
/*public DataTableData<DataManagementPlanListingModel> getPublicPaged(DataManagmentPlanPublicTableRequest dataManagementPlanPublicTableRequest, String fieldsGroup, Principal principal) throws Exception {
dataManagementPlanPublicTableRequest.setQuery(databaseRepository.getDmpDao().asQueryable().withHint(HintedModelFactory.getHint(DataManagementPlanListingModel.class)));
QueryableList<DMP> items = dataManagementPlanPublicTableRequest.applyCriteria();

58
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetManager.java
View File

@ -110,9 +110,10 @@ public class DatasetManager {
private UserManager userManager;
private ConfigLoader configLoader;
private Environment environment;
private final MetricsManager metricsManager;
@Autowired
public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment) {
public DatasetManager(ApiContext apiContext, UserManager userManager, ConfigLoader configLoader, Environment environment, MetricsManager metricsManager) {
this.apiContext = apiContext;
this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
this.datasetRepository = apiContext.getOperationsContext().getElasticRepository().getDatasetRepository();
@ -120,6 +121,7 @@ public class DatasetManager {
this.userManager = userManager;
this.configLoader = configLoader;
this.environment = environment;
this.metricsManager = metricsManager;
}
public DataTableData<DatasetListingModel> getPaged(DatasetTableRequest datasetTableRequest, Principal principal) throws Exception {
@ -156,17 +158,13 @@ public class DatasetManager {
datasetCriteria.setSize(datasetTableRequest.getLength());
List<eu.eudat.elastic.entities.Dataset> datasets;
try {
long start = System.currentTimeMillis();
datasets = datasetRepository.exists() ?
datasetRepository.queryIds(datasetCriteria) : null;
long end = System.currentTimeMillis();
logger.info("Elastic took " + (end - start)/ 1000f + "s");
} catch (Exception ex) {
logger.warn(ex.getMessage(), ex);
datasets = null;
}
long start = System.currentTimeMillis();
UserInfo userInfo = builderFactory.getBuilder(UserInfoBuilder.class).id(principal.getId()).build();
// QueryableList<eu.eudat.data.entities.Dataset> items = databaseRepository.getDatasetDao().getWithCriteria(datasetTableRequest.getCriteria()).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
QueryableList<eu.eudat.data.entities.Dataset> items;
@ -212,26 +210,10 @@ public class DatasetManager {
DataTableData<DatasetListingModel> dataTable = new DataTableData<>();
/* long countStart = System.currentTimeMillis();
List<UUID> pagedIds = pagedItems.toList().stream().map(Dataset::getId).collect(Collectors.toList());
long countEnd = System.currentTimeMillis();
logger.info("Paged Ids took " + (countEnd - countStart)/ 1000f + "s");*/
dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList()));
dataTable.setTotalCount(authItems.count());
// items = databaseRepository.getDatasetDao().asQueryable().where((builder, root) -> root.get("id").in(pagedIds)).withHint(HintedModelFactory.getHint(DatasetListingModel.class));
// PaginationManager.applyOrder(items, datasetTableRequest);
long dataStart = System.currentTimeMillis();
dataTable.setData(pagedItems.select(this::mapModel).stream().filter(Objects::nonNull).collect(Collectors.toList()));
long dataEnd = System.currentTimeMillis();
logger.info("Data retrieval took " + (dataEnd - dataStart)/1000f + "s" );
long end = System.currentTimeMillis();
logger.info("DB took " + (end - start)/ 1000f + "s");
//CompletableFuture.allOf(itemsFuture, countFuture).join();
return dataTable;
}
@ -638,7 +620,7 @@ public class DatasetManager {
sendNotification = true;
}
} else {
MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
}
if (dmp.getStatus().equals(DMP.DMPStatus.FINALISED.getValue()) && datasetWizardModel.getId() != null)
throw new Exception("DMP is finalized, therefore Dataset cannot be edited.");
@ -908,7 +890,7 @@ public class DatasetManager {
if (dataset.getStatus() != eu.eudat.data.entities.Dataset.Status.FINALISED.getValue())
throw new Exception("You cannot make public a Dataset That Has not Been Finalised");
datasetDao.createOrUpdate(dataset);
MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.PUBLISHED);
}
public ResponseEntity<byte[]> getDocument(String id, VisibilityRuleService visibilityRuleService, String contentType, Principal principal) throws IllegalAccessException, IOException, InstantiationException {
@ -997,7 +979,7 @@ public class DatasetManager {
createServicesIfTheyDontExist(entity);
createExternalDatasetsIfTheyDontExist(entity);
MetricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
metricsManager.increaseValue(MetricNames.DATASET, 1, MetricNames.DRAFT);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().createOrUpdate(entity);
}
@ -1112,7 +1094,7 @@ public class DatasetManager {
if (!tagNodes.isEmpty()) {
tagNodes.forEach(node -> {
JsonNode value = node.get("value");
String stringValue = value.asText().replaceAll("=", ":");
String stringValue = value.toString().replaceAll("=", ":");
JSONArray values = new JSONArray(stringValue);
if (values != null) {
values.iterator().forEachRemaining(element -> {
@ -1155,28 +1137,4 @@ public class DatasetManager {
listingModel.setProfileLatestVersion(islast);*/
return listingModel;
}
public long countAllDraft() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllFinalized() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllPublic() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setIsPublic(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllWithDoi() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setHasDoi(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
}

29
dmp-backend/web/src/main/java/eu/eudat/logic/managers/DatasetProfileManager.java
View File

@ -60,12 +60,14 @@ public class DatasetProfileManager {
private DatabaseRepository databaseRepository;
private Environment environment;
private List<String> cache;
private final MetricsManager metricsManager;
@Autowired
public DatasetProfileManager(ApiContext apiContext, Environment environment) {
public DatasetProfileManager(ApiContext apiContext, Environment environment, MetricsManager metricsManager) {
this.apiContext = apiContext;
this.databaseRepository = apiContext.getOperationsContext().getDatabaseRepository();
this.environment = environment;
this.metricsManager = metricsManager;
this.cache = new ArrayList<>();
}
@ -93,7 +95,7 @@ public class DatasetProfileManager {
DatasetProfile profile = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().find(UUID.fromString(id));
apiContext.getOperationsContext().getDatabaseRepository().detachEntity(profile);
profile.setId(null);
MetricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
return profile;
}
@ -234,7 +236,7 @@ public class DatasetProfileManager {
File localFile = convert(multiPartFile);
eu.eudat.logic.utilities.documents.xml.datasetProfileXml.datasetProfileModel.DatasetProfile profile = xmlBuilder.build(localFile);
Files.deleteIfExists(localFile.toPath());
MetricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
metricsManager.increaseValue(MetricNames.DATASET_TEMPLATE, 1, MetricNames.TOTAL);
return profile;
} catch (IOException e) {
logger.error(e.getMessage(), e);
@ -317,25 +319,4 @@ public class DatasetProfileManager {
}).collect(Collectors.toList()));
}
}
public long countAlldraft() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
public long countAllFinalized() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
@Transactional
public long countAllUsed() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
criteria.setAllVersions(false);
List<DatasetProfile> datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
return datasetProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).filter(datasetProfile -> !datasetProfile.getDataset().isEmpty()).count();
}
}

198
dmp-backend/web/src/main/java/eu/eudat/logic/managers/MetricsManager.java
View File

@ -1,6 +1,12 @@
package eu.eudat.logic.managers;
import eu.eudat.types.MetricNames;
import eu.eudat.data.dao.criteria.*;
import eu.eudat.data.entities.Dataset;
import eu.eudat.data.entities.DatasetProfile;
import eu.eudat.logic.services.ApiContext;
import eu.eudat.logic.utilities.helpers.StreamDistinctBy;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import io.prometheus.client.Gauge;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -14,32 +20,16 @@ import javax.transaction.Transactional;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
public class MetricsManager {
private final static Logger logger = LoggerFactory.getLogger(MetricsManager.class);
private static Map<String, Gauge> gauges = Stream.of( new Object[][]{
{MetricNames.DMP, Gauge.build().name(MetricNames.DMP).help("Number of managed DMPs").labelNames("status").register()},
private final Map<String, Gauge> gauges;
{MetricNames.FUNDERS, Gauge.build().name(MetricNames.FUNDERS).help("Number of registered Funders").register()},
{MetricNames.GRANTS, Gauge.build().name(MetricNames.GRANTS).help("Number of registered Grants").register()},
{MetricNames.PROJECT, Gauge.build().name(MetricNames.PROJECT).help("Number of registered Projects").register()},
{MetricNames.RESEARCHER, Gauge.build().name(MetricNames.RESEARCHER).help("Number of Colaborators/Researchers").register()},
{MetricNames.DATASET, Gauge.build().name(MetricNames.DATASET).help("Number of managed Dataset Descriptions").labelNames("status").register()},
{MetricNames.DATASET_TEMPLATE, Gauge.build().name(MetricNames.DATASET_TEMPLATE).help("Number of dataset Templates").labelNames("status").register()},
{MetricNames.USERS, Gauge.build().name(MetricNames.USERS).help("Number of users").labelNames("type").register()},
{MetricNames.LANGUAGES, Gauge.build().name(MetricNames.LANGUAGES).help("Number of Languages").register()},
}).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
public static void increaseValue(String name, int amount, String label) {
public void increaseValue(String name, int amount, String label) {
if(label != null) {
gauges.get(name).labels(label).inc(amount);
@ -48,7 +38,7 @@ public class MetricsManager {
}
}
public static void decreaseValue(String name, int amount, String label) {
public void decreaseValue(String name, int amount, String label) {
if(label != null) {
gauges.get(name).labels(label).dec(amount);
} else {
@ -56,7 +46,7 @@ public class MetricsManager {
}
}
public static Integer getValue(String name, String label) {
public Integer getValue(String name, String label) {
if(label != null) {
return Double.valueOf(gauges.get(name).labels(label).get()).intValue();
} else {
@ -64,7 +54,7 @@ public class MetricsManager {
}
}
public static void calculateValue(String name, int amount, String label) {
public void calculateValue(String name, int amount, String label) {
Integer orig = getValue(name, label);
int diff = orig - amount;
if (diff != 0) {
@ -76,19 +66,34 @@ public class MetricsManager {
}
}
private final DatasetManager datasetManager;
private final DataManagementPlanManager dataManagementPlanManager;
private final DatasetProfileManager datasetProfileManager;
private final ApiContext apiContext;
private final UserManager userManager;
private final Environment environment;
@Autowired
public MetricsManager(DatasetManager datasetManager, DataManagementPlanManager dataManagementPlanManager, DatasetProfileManager datasetProfileManager, UserManager userManager, Environment environment) {
this.datasetManager = datasetManager;
this.dataManagementPlanManager = dataManagementPlanManager;
this.datasetProfileManager = datasetProfileManager;
public MetricsManager(ApiContext apiContext, UserManager userManager, Environment environment, PrometheusMeterRegistry registry) {
this.apiContext = apiContext;
this.userManager = userManager;
this.environment = environment;
registry.clear();
this.gauges = Stream.of( new Object[][]{
{MetricNames.DMP, Gauge.build().name("argos_managed_dmps").help("Number of managed DMPs").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.FUNDERS, Gauge.build().name("argos_funders").help("Number of registered Funders").register(registry.getPrometheusRegistry())},
{MetricNames.GRANTS, Gauge.build().name("argos_grants").help("Number of registered Grants").register(registry.getPrometheusRegistry())},
{MetricNames.PROJECT, Gauge.build().name("argos_projects").help("Number of registered Projects").register(registry.getPrometheusRegistry())},
{MetricNames.RESEARCHER, Gauge.build().name("argos_researchers").help("Number of Colaborators/Researchers").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET, Gauge.build().name("argos_managed_dataset_descriptions").help("Number of managed Dataset Descriptions").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.DATASET_TEMPLATE, Gauge.build().name("argos_dataset_templates").help("Number of dataset Templates").labelNames("status").register(registry.getPrometheusRegistry())},
{MetricNames.USERS, Gauge.build().name("argos_users").help("Number of users").labelNames("type").register(registry.getPrometheusRegistry())},
{MetricNames.LANGUAGES, Gauge.build().name("argos_languages").help("Number of Languages").register(registry.getPrometheusRegistry())},
}).collect(Collectors.toMap(data -> (String)data[0], data -> (Gauge) data[1]));
}
@PostConstruct
@ -96,32 +101,131 @@ public class MetricsManager {
@Scheduled(initialDelay = 1000 * 60 * 60, fixedDelay = 1000 * 60 * 60)
public void init() throws IOException {
logger.info("Start calculating Metrics");
calculateValue(MetricNames.DMP, (int) dataManagementPlanManager.countAllDrafts(), MetricNames.DRAFT);
calculateValue(MetricNames.DMP, (int) dataManagementPlanManager.countAllFinalized(), MetricNames.FINALIZED);
calculateValue(MetricNames.DMP, (int) dataManagementPlanManager.countAllPublished(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DMP, (int) dataManagementPlanManager.countAllDoied(), MetricNames.DOIED);
calculateValue("argos_managed_dmps", (int) countAllDraftDMPs(), "draft");
calculateValue("argos_managed_dmps", (int) countAllFinalizedDMPs(), "finalized");
calculateValue("argos_managed_dmps", (int) countAllPublishedDMPs(), "published");
calculateValue("argos_managed_dmps", (int) countAllDoiedDMPs(), "doied");
calculateValue(MetricNames.FUNDERS, (int) dataManagementPlanManager.countAllFunders(), null);
calculateValue(MetricNames.GRANTS, (int) dataManagementPlanManager.countAllGrants(), null);
calculateValue(MetricNames.PROJECT, (int) dataManagementPlanManager.countAllProjects(), null);
calculateValue(MetricNames.RESEARCHER, (int) dataManagementPlanManager.countAllResearchers(), null);
calculateValue("argos_funders", (int) countAllFunders(), null);
calculateValue("argos_grants", (int) countAllGrants(), null);
calculateValue("argos_projects", (int) countAllProjects(), null);
calculateValue("argos_researchers", (int) countAllResearchers(), null);
calculateValue(MetricNames.DATASET, (int) datasetManager.countAllDraft(), MetricNames.DRAFT);
calculateValue(MetricNames.DATASET, (int) datasetManager.countAllFinalized(), MetricNames.FINALIZED);
calculateValue(MetricNames.DATASET, (int) datasetManager.countAllPublic(), MetricNames.PUBLISHED);
calculateValue(MetricNames.DATASET, (int) datasetManager.countAllWithDoi(), MetricNames.DOIED);
calculateValue("argos_managed_dataset_descriptions", (int) countAllDraftDatasets(), "draft");
calculateValue("argos_managed_dataset_descriptions", (int) countAllFinalizedDatasets(), "finalized");
calculateValue("argos_managed_dataset_descriptions", (int) countAllPublicDatasets(), "published");
calculateValue("argos_managed_dataset_descriptions", (int) countAllDatasetsWithDoi(), "doied");
calculateValue(MetricNames.DATASET_TEMPLATE, (int) datasetProfileManager.countAlldraft(), MetricNames.DRAFT);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) datasetProfileManager.countAllFinalized(), MetricNames.ACTIVE);
calculateValue(MetricNames.DATASET_TEMPLATE, (int) datasetProfileManager.countAllUsed(), MetricNames.USED);
calculateValue("argos_dataset_templates", (int) countAllDraftTemplates(), "draft");
calculateValue("argos_dataset_templates", (int) countAllFinalizedTemplates(), "active");
calculateValue("argos_dataset_templates", (int) countAllUsedTemplates(), "used");
calculateValue(MetricNames.USERS, (int) userManager.countActiveUsers().intValue(), MetricNames.LOGGEDIN);
calculateValue(MetricNames.USERS, (int) userManager.countAllUsers().intValue(), MetricNames.TOTAL);
calculateValue("argos_users", (int) userManager.countActiveUsers().intValue(), "loggedin");
calculateValue("argos_users", (int) userManager.countAllUsers().intValue(), "total");
long files = Files.list(Paths.get(this.environment.getProperty("userguide.path"))).count();
calculateValue(MetricNames.LANGUAGES, (int) files, null);
calculateValue("argos_languages", (int) files, null);
logger.info("Metrics calculation Completed");
}
private long countAllDraftDMPs() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllFinalizedDMPs() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllPublishedDMPs() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setIsPublic(true);
criteria.setOnlyPublic(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllDoiedDMPs() {
DataManagementPlanCriteria criteria = new DataManagementPlanCriteria();
criteria.setHasDoi(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDmpDao().getWithCriteria(criteria).count();
}
private long countAllResearchers() {
ResearcherCriteria criteria = new ResearcherCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getResearcherDao().getWithCriteria(criteria).count();
}
private long countAllProjects() {
ProjectCriteria criteria = new ProjectCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getProjectDao().getWithCritetia(criteria).count();
}
private long countAllFunders() {
FunderCriteria criteria = new FunderCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getFunderDao().getWithCritetia(criteria).count();
}
private long countAllGrants() {
GrantCriteria criteria = new GrantCriteria();
return apiContext.getOperationsContext().getDatabaseRepository().getGrantDao().getWithCriteria(criteria).count();
}
public long countAllDraftDatasets() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllFinalizedDatasets() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllPublicDatasets() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setIsPublic(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllDatasetsWithDoi() {
eu.eudat.data.dao.criteria.DatasetCriteria criteria = new eu.eudat.data.dao.criteria.DatasetCriteria();
criteria.setHasDoi(true);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(criteria).count();
}
public long countAllDraftTemplates() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(0);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
public long countAllFinalizedTemplates() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
return apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).count();
}
@Transactional
public long countAllUsedTemplates() {
DatasetProfileCriteria criteria = new DatasetProfileCriteria();
criteria.setStatus(1);
criteria.setAllVersions(false);
List<DatasetProfile> datasetProfiles = apiContext.getOperationsContext().getDatabaseRepository().getDatasetProfileDao().getWithCriteria(criteria).toList();
List<DatasetProfile> filteredProfiles = new ArrayList<>();
for (DatasetProfile datasetProfile : datasetProfiles) {
DatasetCriteria datasetCriteria = new DatasetCriteria();
datasetCriteria.setDatasetTemplates(Collections.singletonList(datasetProfile.getId()));
Long datasetCount = apiContext.getOperationsContext().getDatabaseRepository().getDatasetDao().getWithCriteria(datasetCriteria).count();
if (datasetCount > 0) {
filteredProfiles.add(datasetProfile);
}
}
return filteredProfiles.stream().filter(StreamDistinctBy.distinctByKey(DatasetProfile::getId)).count();
}
}

2
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View File

@ -459,7 +459,7 @@ public class RemoteFetcher {
try {
String filePath = Paths.get(path).toUri().toURL().toString();
ObjectMapper mapper = new ObjectMapper();
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, Object>>>(){});
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, String>>>(){});
return searchListMap(internalResults, query);
} catch (Exception e) {
logger.error(e.getMessage(), e);

15
dmp-backend/web/src/main/java/eu/eudat/models/data/dmp/DataManagementPlanNewVersionModel.java
View File

@ -7,7 +7,10 @@ import eu.eudat.models.data.funder.FunderDMPEditorModel;
import eu.eudat.models.data.grant.GrantDMPEditorModel;
import eu.eudat.models.data.project.ProjectDMPEditorModel;
import eu.eudat.models.data.userinfo.UserInfo;
import eu.eudat.models.data.userinfo.UserListingModel;
import net.minidev.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
import java.util.stream.Collectors;
@ -16,6 +19,8 @@ import java.util.stream.Collectors;
* Created by ikalyvas on 2/5/2018.
*/
public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataManagementPlanNewVersionModel> {
private static final Logger logger = LoggerFactory.getLogger(DataManagementPlanNewVersionModel.class);
private UUID id;
private String label;
private UUID groupId;
@ -26,7 +31,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
private eu.eudat.models.data.grant.GrantDMPEditorModel grant;
private List<Organisation> organisations;
private List<Researcher> researchers;
private List<UserDMP> associatedUsers;
private List<UserListingModel> associatedUsers;
private eu.eudat.models.data.userinfo.UserInfo creator;
private Date created;
private List<Dataset> datasets;
@ -104,10 +109,10 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
this.researchers = researchers;
}
public List<UserDMP> getAssociatedUsers() {
public List<UserListingModel> getAssociatedUsers() {
return associatedUsers;
}
public void setAssociatedUsers(List<UserDMP> associatedUsers) {
public void setAssociatedUsers(List<UserListingModel> associatedUsers) {
this.associatedUsers = associatedUsers;
}
@ -163,7 +168,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
public DMP toDataModel() throws Exception {
DMP entity = new DMP();
entity.setId(this.id);
entity.setUsers(new HashSet<>(new ArrayList<>(this.associatedUsers)));
// entity.setUsers(new HashSet<>(new ArrayList<>(this.associatedUsers)));
entity.setDescription(this.description);
entity.setStatus((short) this.status);
entity.setGroupId(this.groupId == null ? UUID.randomUUID() : this.groupId);
@ -227,7 +232,7 @@ public class DataManagementPlanNewVersionModel implements DataModel<DMP, DataMan
funder.setModified(new Date());
entity.getGrant().setFunder(funder);
} else if (this.funder.getExistFunder() != null && this.funder.getLabel() == null){
} else if (this.funder.getExistFunder() != null && this.funder.getLabel() == null) {
entity.getGrant().setFunder(this.funder.getExistFunder().toDataModel());
entity.getGrant().getFunder().setType(Funder.FunderType.EXTERNAL.getValue());
}

14
dmp-backend/web/src/main/resources/config/application.properties
View File

@ -5,6 +5,20 @@ logging.file=/logs/spring-boot-logging.log
spring.profiles.active=devel
eu.eudat.logic.proxy.allowed.host=https://eestore.paas2.uninett.no
####################Metrics##############
management.endpoint.metrics.enabled=false
management.metrics.enable.http=false
management.metrics.enable.jvm=false
management.metrics.enable.jdbc=false
management.metrics.enable.tomcat=false
management.metrics.enable.logback=false
management.metrics.enable.hikaricp=false
management.metrics.enable.cache=false
management.endpoints.web.base-path=/
management.endpoints.web.exposure.include=prometheus
management.endpoint.prometheus.enabled=true
management.metrics.export.prometheus.enabled=true
####################INVITATION MAIL CONFIGURATIONS##############
####################GENERIC MAIL CONFIGURATIONS#################
mail.subject=Invitation to DMP Plan {dmpname}